Mercurial > hg
annotate hgext/lfs/blobstore.py @ 41731:92055d539e49
tests: conditionalize msys path mangling in test-bundle.t
This broke in 252cc56c9ff6 when the variables were printed one per line. The
only reason I can think of is that MSYS knew the former string couldn't be a
list of paths, but thinks this form is.
author | Matt Harbison <matt_harbison@yahoo.com> |
---|---|
date | Sat, 16 Feb 2019 22:03:58 -0500 |
parents | 698667eb7523 |
children | 2372284d9457 |
rev | line source |
---|---|
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
9 |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
10 import contextlib |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
11 import errno |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
12 import hashlib |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
14 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
15 import re |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
16 import socket |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
17 |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
18 from mercurial.i18n import _ |
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
19 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
20 from mercurial import ( |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
21 encoding, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
22 error, |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
23 node, |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
24 pathutil, |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
25 pycompat, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
26 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
27 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 vfs as vfsmod, |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
29 worker, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
30 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
31 |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
32 from mercurial.utils import ( |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
33 stringutil, |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
34 ) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
35 |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
36 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
37 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
38 # 64 bytes for SHA256 |
36455
9e3cb58c7ab3
py3: make sure regexes are bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35927
diff
changeset
|
39 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
40 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
41 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
42 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
43 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
44 if not _lfsre.match(path): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
45 raise error.ProgrammingError(b'unexpected lfs path: %s' % path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
46 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
47 |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
48 def walk(self, path=None, onerror=None): |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
49 """Yield (dirpath, [], oids) tuple for blobs under path |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
50 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
51 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
52 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
53 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
54 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
55 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
56 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
57 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
58 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
59 for dirpath, dirs, files in os.walk(self.reljoin(self.base, path |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
60 or b''), |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
61 onerror=onerror): |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
62 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
63 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
64 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
65 if len(dirpath) == 2: |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
66 oids.extend([dirpath + f for f in files |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
67 if _lfsre.match(dirpath + f)]) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
68 |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
69 yield ('', [], oids) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
70 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
71 class nullvfs(lfsvfs): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
72 def __init__(self): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
73 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
74 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
75 def exists(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
76 return False |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
77 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
78 def read(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
79 # store.read() calls into here if the blob doesn't exist in its |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
80 # self.vfs. Raise the same error as a normal vfs when asked to read a |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
81 # file that doesn't exist. The only difference is the full file path |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
82 # isn't available in the error. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
83 raise IOError(errno.ENOENT, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
84 pycompat.sysstr(b'%s: No such file or directory' % oid)) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
85 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
86 def walk(self, path=None, onerror=None): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
87 return (b'', [], []) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
88 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
89 def write(self, oid, data): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
90 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
91 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
92 class filewithprogress(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
93 """a file-like object that supports __len__ and read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
94 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
95 Useful to provide progress information for how many bytes are read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
96 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
97 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
98 def __init__(self, fp, callback): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
99 self._fp = fp |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
100 self._callback = callback # func(readsize) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
101 fp.seek(0, os.SEEK_END) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
102 self._len = fp.tell() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
103 fp.seek(0) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
104 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
105 def __len__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
106 return self._len |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
107 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
108 def read(self, size): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
109 if self._fp is None: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
110 return b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
111 data = self._fp.read(size) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 if data: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 if self._callback: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
114 self._callback(len(data)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
115 else: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
116 self._fp.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
117 self._fp = None |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
118 return data |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
119 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
120 class local(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
121 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
122 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
123 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
124 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
125 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
126 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
127 def __init__(self, repo): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
128 fullpath = repo.svfs.join(b'lfs/objects') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
129 self.vfs = lfsvfs(fullpath) |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
130 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
131 if repo.ui.configbool(b'experimental', b'lfs.disableusercache'): |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
132 self.cachevfs = nullvfs() |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
133 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
134 usercache = lfutil._usercachedir(repo.ui, b'lfs') |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
135 self.cachevfs = lfsvfs(usercache) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
136 self.ui = repo.ui |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
137 |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
138 def open(self, oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
139 """Open a read-only file descriptor to the named blob, in either the |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
140 usercache or the local store.""" |
35537
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
141 # The usercache is the most likely place to hold the file. Commit will |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
142 # write to both it and the local store, as will anything that downloads |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
143 # the blobs. However, things like clone without an update won't |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
144 # populate the local store. For an init + push of a local clone, |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
145 # the usercache is the only place it _could_ be. If not present, the |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
146 # missing file msg here will indicate the local repo, not the usercache. |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
147 if self.cachevfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
148 return self.cachevfs(oid, b'rb') |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
149 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
150 return self.vfs(oid, b'rb') |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
151 |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
152 def download(self, oid, src): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
153 """Read the blob from the remote source in chunks, verify the content, |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
154 and write to this local blobstore.""" |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
155 sha256 = hashlib.sha256() |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
156 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
157 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
158 for chunk in util.filechunkiter(src, size=1048576): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
159 fp.write(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
160 sha256.update(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
161 |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
162 realoid = node.hex(sha256.digest()) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
163 if realoid != oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
164 raise LfsCorruptionError(_(b'corrupt remote lfs object: %s') |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
165 % oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
166 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
167 self._linktousercache(oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
168 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
169 def write(self, oid, data): |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
170 """Write blob to local blobstore. |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
171 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
172 This should only be called from the filelog during a commit or similar. |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
173 As such, there is no need to verify the data. Imports from a remote |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
174 store must use ``download()`` instead.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
175 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
176 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
177 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
178 self._linktousercache(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
179 |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
180 def linkfromusercache(self, oid): |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
181 """Link blobs found in the user cache into this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
182 |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
183 The server module needs to do this when it lets the client know not to |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
184 upload the blob, to ensure it is always available in this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
185 Normally this is done implicitly when the client reads or writes the |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
186 blob, but that doesn't happen when the server tells the client that it |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
187 already has the blob. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
188 """ |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
189 if (not isinstance(self.cachevfs, nullvfs) |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
190 and not self.vfs.exists(oid)): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
191 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
192 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
193 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
194 def _linktousercache(self, oid): |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
195 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
196 # the local store on success, but truncate, write and link on failure? |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
197 if (not self.cachevfs.exists(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
198 and not isinstance(self.cachevfs, nullvfs)): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
199 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
200 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
201 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
202 def read(self, oid, verify=True): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
203 """Read blob from local blobstore.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
204 if not self.vfs.exists(oid): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
205 blob = self._read(self.cachevfs, oid, verify) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
206 |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
207 # Even if revlog will verify the content, it needs to be verified |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
208 # now before making the hardlink to avoid propagating corrupt blobs. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
209 # Don't abort if corruption is detected, because `hg verify` will |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
210 # give more useful info about the corruption- simply don't add the |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
211 # hardlink. |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
212 if verify or node.hex(hashlib.sha256(blob).digest()) == oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
213 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
214 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
215 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
216 self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
217 blob = self._read(self.vfs, oid, verify) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
218 return blob |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
219 |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
220 def _read(self, vfs, oid, verify): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
221 """Read blob (after verifying) from the given store""" |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
222 blob = vfs.read(oid) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
223 if verify: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
224 _verify(oid, blob) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
225 return blob |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
226 |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
227 def verify(self, oid): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
228 """Indicate whether or not the hash of the underlying file matches its |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
229 name.""" |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
230 sha256 = hashlib.sha256() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
231 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
232 with self.open(oid) as fp: |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
233 for chunk in util.filechunkiter(fp, size=1048576): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
234 sha256.update(chunk) |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
235 |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
236 return oid == node.hex(sha256.digest()) |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
237 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
238 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
239 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
240 False otherwise.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
241 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
242 |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
243 def _urlerrorreason(urlerror): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
244 '''Create a friendly message for the given URLError to be used in an |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
245 LfsRemoteError message. |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
246 ''' |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
247 inst = urlerror |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
248 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
249 if isinstance(urlerror.reason, Exception): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
250 inst = urlerror.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
251 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
252 if util.safehasattr(inst, 'reason'): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
253 try: # usually it is in the form (errno, strerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
254 reason = inst.reason.args[1] |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
255 except (AttributeError, IndexError): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
256 # it might be anything, for example a string |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
257 reason = inst.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
258 if isinstance(reason, pycompat.unicode): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
259 # SSLError of Python 2.7.9 contains a unicode |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
260 reason = encoding.unitolocal(reason) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
261 return reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
262 elif getattr(inst, "strerror", None): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
263 return encoding.strtolocal(inst.strerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
264 else: |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
265 return stringutil.forcebytestr(urlerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
266 |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
267 class lfsauthhandler(util.urlreq.basehandler): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
268 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
269 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
270 def http_error_401(self, req, fp, code, msg, headers): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
271 """Enforces that any authentication performed is HTTP Basic |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
272 Authentication. No authentication is also acceptable. |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
273 """ |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
274 authreq = headers.get(r'www-authenticate', None) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
275 if authreq: |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
276 scheme = authreq.split()[0] |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
277 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
278 if scheme.lower() != r'basic': |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
279 msg = _(b'the server must support Basic Authentication') |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
280 raise util.urlerr.httperror(req.get_full_url(), code, |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
281 encoding.strfromlocal(msg), headers, |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
282 fp) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
283 return None |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
284 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
285 class _gitlfsremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
286 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
287 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
288 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
289 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
290 baseurl, authinfo = url.authinfo() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
291 self.baseurl = baseurl.rstrip(b'/') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
292 useragent = repo.ui.config(b'experimental', b'lfs.user-agent') |
35440
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
293 if not useragent: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
294 useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
35439
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35433
diff
changeset
|
295 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
296 self.urlopener.add_handler(lfsauthhandler()) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
297 self.retry = ui.configint(b'lfs', b'retry') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
298 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
299 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
300 """Batch upload from local to remote blobstore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
301 self._batch(_deduplicate(pointers), fromstore, b'upload') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
302 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
303 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
304 """Batch download from remote to local blostore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
305 self._batch(_deduplicate(pointers), tostore, b'download') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
306 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
307 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
308 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
309 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
310 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
311 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
312 """ |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
313 objects = [{r'oid': pycompat.strurl(p.oid()), |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
314 r'size': p.size()} for p in pointers] |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
315 requestdata = pycompat.bytesurl(json.dumps({ |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
316 r'objects': objects, |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
317 r'operation': pycompat.strurl(action), |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
318 })) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
319 url = b'%s/objects/batch' % self.baseurl |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
320 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
321 batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
322 batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
323 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
324 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
325 rawjson = rsp.read() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
326 except util.urlerr.httperror as ex: |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
327 hints = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
328 400: _(b'check that lfs serving is enabled on %s and "%s" is ' |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
329 b'supported') % (self.baseurl, action), |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
330 404: _(b'the "lfs.url" config may be used to override %s') |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
331 % self.baseurl, |
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
332 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
333 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
334 raise LfsRemoteError( |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
335 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
336 hint=hint) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
337 except util.urlerr.urlerror as ex: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
338 hint = (_(b'the "lfs.url" config may be used to override %s') |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
339 % self.baseurl) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
340 raise LfsRemoteError(_(b'LFS error: %s') % _urlerrorreason(ex), |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
341 hint=hint) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
342 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
343 response = json.loads(rawjson) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
344 except ValueError: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
345 raise LfsRemoteError(_(b'LFS server returns invalid JSON: %s') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
346 % rawjson.encode("utf-8")) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
347 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
348 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
349 self.ui.debug(b'Status: %d\n' % rsp.status) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
350 # lfs-test-server and hg serve return headers in different order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
351 headers = pycompat.bytestr(rsp.info()).strip() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
352 self.ui.debug(b'%s\n' |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
353 % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
354 |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
355 if r'objects' in response: |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
356 response[r'objects'] = sorted(response[r'objects'], |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
357 key=lambda p: p[r'oid']) |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
358 self.ui.debug(b'%s\n' |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
359 % pycompat.bytesurl( |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
360 json.dumps(response, indent=2, |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
361 separators=(r'', r': '), |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
362 sort_keys=True))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
363 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
364 def encodestr(x): |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
365 if isinstance(x, pycompat.unicode): |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
366 return x.encode(u'utf-8') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
367 return x |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
368 |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
369 return pycompat.rapply(encodestr, response) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
370 |
35666 | 371 def _checkforservererror(self, pointers, responses, action): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
372 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
373 |
35694
8a23082f4d93
lfs: correct documentation typo
Matt Harbison <matt_harbison@yahoo.com>
parents:
35666
diff
changeset
|
374 Raises LfsRemoteError if any objects have an error""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
375 for response in responses: |
35666 | 376 # The server should return 404 when objects cannot be found. Some |
377 # server implementation (ex. lfs-test-server) does not set "error" | |
378 # but just removes "download" from "actions". Treat that case | |
379 # as the same as 404 error. | |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
380 if b'error' not in response: |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
381 if (action == b'download' |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
382 and action not in response.get(b'actions', [])): |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
383 code = 404 |
35695
dd672e3d059f
lfs: raise an error if the server sends an unsolicited oid
Matt Harbison <matt_harbison@yahoo.com>
parents:
35694
diff
changeset
|
384 else: |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
385 continue |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
386 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
387 # An error dict without a code doesn't make much sense, so |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
388 # treat as a server error. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
389 code = response.get(b'error').get(b'code', 500) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
390 |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
391 ptrmap = {p.oid(): p for p in pointers} |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
392 p = ptrmap.get(response[b'oid'], None) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
393 if p: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
394 filename = getattr(p, 'filename', b'unknown') |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
395 errors = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
396 404: b'The object does not exist', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
397 410: b'The object was removed by the owner', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
398 422: b'Validation error', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
399 500: b'Internal server error', |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
400 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
401 msg = errors.get(code, b'status code %d' % code) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
402 raise LfsRemoteError(_(b'LFS server error for "%s": %s') |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
403 % (filename, msg)) |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
404 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
405 raise LfsRemoteError( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
406 _(b'LFS server error. Unsolicited response for oid %s') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
407 % response[b'oid']) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
408 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
409 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
410 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
411 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
412 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
413 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
414 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
415 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
416 # Scan errors from objects - fail early |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
417 objects = response.get(b'objects', []) |
35666 | 418 self._checkforservererror(pointers, objects, action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
419 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
420 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
421 # objects which exist in the server. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
422 filteredobjects = [o for o in objects |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
423 if action in o.get(b'actions', [])] |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
424 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
425 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
426 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
427 def _basictransfer(self, obj, action, localstore): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
428 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
429 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
430 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
431 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
432 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
433 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
434 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
435 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
436 """ |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
437 oid = obj[b'oid'] |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
438 href = obj[b'actions'][action].get(b'href') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
439 headers = obj[b'actions'][action].get(b'header', {}).items() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
440 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
441 request = util.urlreq.request(pycompat.strurl(href)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
442 if action == b'upload': |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
443 # If uploading blobs, read data from local blobstore. |
37217
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
444 if not localstore.verify(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
445 raise error.Abort(_(b'detected corrupt lfs object: %s') % oid, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
446 hint=_(b'run hg verify')) |
35526
e8f80529abeb
lfs: use the local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35525
diff
changeset
|
447 request.data = filewithprogress(localstore.open(oid), None) |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
448 request.get_method = lambda: r'PUT' |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
449 request.add_header(r'Content-Type', r'application/octet-stream') |
41440
1bc01490178a
lfs: explicitly add the Content-Length header when uploading blobs, for py3
Matt Harbison <matt_harbison@yahoo.com>
parents:
41431
diff
changeset
|
450 request.add_header(r'Content-Length', len(request.data)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
451 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
452 for k, v in headers: |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
453 request.add_header(pycompat.strurl(k), pycompat.strurl(v)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
454 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
455 response = b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
456 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
457 with contextlib.closing(self.urlopener.open(request)) as req: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
458 ui = self.ui # Shorten debug lines |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
459 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
460 ui.debug(b'Status: %d\n' % req.status) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
461 # lfs-test-server and hg serve return headers in different |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
462 # order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
463 headers = pycompat.bytestr(req.info()).strip() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
464 ui.debug(b'%s\n' |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
465 % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
466 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
467 if action == b'download': |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
468 # If downloading blobs, store downloaded data to local |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
469 # blobstore |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
470 localstore.download(oid, req) |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
471 else: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
472 while True: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
473 data = req.read(1048576) |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
474 if not data: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
475 break |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
476 response += data |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
477 if response: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
478 ui.debug(b'lfs %s response: %s' % (action, response)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
479 except util.urlerr.httperror as ex: |
35734
b4e1d0654736
lfs: dump the full response on httperror in debug mode
Matt Harbison <matt_harbison@yahoo.com>
parents:
35733
diff
changeset
|
480 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
481 self.ui.debug(b'%s: %s\n' % (oid, ex.read())) # XXX: also bytes? |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
482 raise LfsRemoteError(_(b'LFS HTTP error: %s (oid=%s, action=%s)') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
483 % (stringutil.forcebytestr(ex), oid, action)) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
484 except util.urlerr.urlerror as ex: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
485 hint = (_(b'attempted connection to %s') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
486 % pycompat.bytesurl(util.urllibcompat.getfullurl(request))) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
487 raise LfsRemoteError(_(b'LFS error: %s') % _urlerrorreason(ex), |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
488 hint=hint) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
489 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
490 def _batch(self, pointers, localstore, action): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
491 if action not in [b'upload', b'download']: |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
492 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
493 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
494 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
495 objects = self._extractobjects(response, pointers, action) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
496 total = sum(x.get(b'size', 0) for x in objects) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
497 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
498 for obj in objects: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
499 sizes[obj.get(b'oid')] = obj.get(b'size', 0) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
500 topic = {b'upload': _(b'lfs uploading'), |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
501 b'download': _(b'lfs downloading')}[action] |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
502 if len(objects) > 1: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
503 self.ui.note(_(b'lfs: need to transfer %d objects (%s)\n') |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
504 % (len(objects), util.bytecount(total))) |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
505 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
506 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
507 for obj in chunk: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
508 objsize = obj.get(b'size', 0) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
509 if self.ui.verbose: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
510 if action == b'download': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
511 msg = _(b'lfs: downloading %s (%s)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
512 elif action == b'upload': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
513 msg = _(b'lfs: uploading %s (%s)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
514 self.ui.note(msg % (obj.get(b'oid'), |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
515 util.bytecount(objsize))) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
516 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
517 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
518 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
519 self._basictransfer(obj, action, localstore) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
520 yield 1, obj.get(b'oid') |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
521 break |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
522 except socket.error as ex: |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
523 if retry > 0: |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
524 self.ui.note( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
525 _(b'lfs: failed: %r (remaining retry %d)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
526 % (stringutil.forcebytestr(ex), retry)) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
527 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
528 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
529 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
530 |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
531 # Until https multiplexing gets sorted out |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
532 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
533 oids = worker.worker(self.ui, 0.1, transfer, (), |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
534 sorted(objects, key=lambda o: o.get(b'oid'))) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
535 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
536 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
537 |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
538 with self.ui.makeprogress(topic, total=total) as progress: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
539 progress.update(0) |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
540 processed = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
541 blobs = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
542 for _one, oid in oids: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
543 processed += sizes[oid] |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
544 blobs += 1 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
545 progress.update(processed) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
546 self.ui.note(_(b'lfs: processed: %s\n') % oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
547 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
548 if blobs > 0: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
549 if action == b'upload': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
550 self.ui.status(_(b'lfs: uploaded %d files (%s)\n') |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
551 % (blobs, util.bytecount(processed))) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
552 elif action == b'download': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
553 self.ui.status(_(b'lfs: downloaded %d files (%s)\n') |
37765
ab04972a33ef
lfs: enable the final download count status message
Matt Harbison <matt_harbison@yahoo.com>
parents:
37692
diff
changeset
|
554 % (blobs, util.bytecount(processed))) |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
555 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
556 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
557 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
558 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
559 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
560 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
561 h.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
562 getattr(h, "close_all", lambda : None)() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
563 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
564 class _dummyremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
565 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
566 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
567 def __init__(self, repo, url): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
568 fullpath = repo.vfs.join(b'lfs', url.path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
569 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
570 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
571 def writebatch(self, pointers, fromstore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
572 for p in _deduplicate(pointers): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
573 content = fromstore.read(p.oid(), verify=True) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
574 with self.vfs(p.oid(), b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
575 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
576 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
577 def readbatch(self, pointers, tostore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
578 for p in _deduplicate(pointers): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
579 with self.vfs(p.oid(), b'rb') as fp: |
35552
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
580 tostore.download(p.oid(), fp) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
581 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
582 class _nullremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
583 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
584 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
585 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
586 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
587 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
588 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
589 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
590 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
591 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
592 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
593 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
594 class _promptremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
595 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
596 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
597 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
598 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
599 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
600 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
601 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
602 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
603 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
604 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
605 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
606 def _prompt(self): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
607 raise error.Abort(_(b'lfs.url needs to be configured')) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
608 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
609 _storemap = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
610 b'https': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
611 b'http': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
612 b'file': _dummyremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
613 b'null': _nullremote, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
614 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
615 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
616 |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
617 def _deduplicate(pointers): |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
618 """Remove any duplicate oids that exist in the list""" |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
619 reduced = util.sortdict() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
620 for p in pointers: |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
621 reduced[p.oid()] = p |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
622 return reduced.values() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
623 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
624 def _verify(oid, content): |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
625 realoid = node.hex(hashlib.sha256(content).digest()) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
626 if realoid != oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
627 raise LfsCorruptionError(_(b'detected corrupt lfs object: %s') % oid, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
628 hint=_(b'run hg verify')) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
629 |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
630 def remote(repo, remote=None): |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
631 """remotestore factory. return a store in _storemap depending on config |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
632 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
633 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
634 infer the endpoint, based on the remote repository using the same path |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
635 adjustments as git. As an extension, 'http' is supported as well so that |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
636 ``hg serve`` works out of the box. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
637 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
638 https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
639 """ |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
640 lfsurl = repo.ui.config(b'lfs', b'url') |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
641 url = util.url(lfsurl or '') |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
642 if lfsurl is None: |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
643 if remote: |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
644 path = remote |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
645 elif util.safehasattr(repo, '_subtoppath'): |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
646 # The pull command sets this during the optional update phase, which |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
647 # tells exactly where the pull originated, whether 'paths.default' |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
648 # or explicit. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
649 path = repo._subtoppath |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
650 else: |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
651 # TODO: investigate 'paths.remote:lfsurl' style path customization, |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
652 # and fall back to inferring from 'paths.remote' if unspecified. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
653 path = repo.ui.config(b'paths', b'default') or b'' |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
654 |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
655 defaulturl = util.url(path) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
656 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
657 # TODO: support local paths as well. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
658 # TODO: consider the ssh -> https transformation that git applies |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
659 if defaulturl.scheme in (b'http', b'https'): |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
660 if defaulturl.path and defaulturl.path[:-1] != b'/': |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
661 defaulturl.path += b'/' |
37691
d241e6632669
lfs: fix the inferred remote store path when using a --prefix
Matt Harbison <matt_harbison@yahoo.com>
parents:
37565
diff
changeset
|
662 defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
663 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
664 url = util.url(bytes(defaulturl)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
665 repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
666 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
667 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
668 if scheme not in _storemap: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
669 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
670 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
671 |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39457
diff
changeset
|
672 class LfsRemoteError(error.StorageError): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
673 pass |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
674 |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
675 class LfsCorruptionError(error.Abort): |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
676 """Raised when a corrupt blob is detected, aborting an operation |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
677 |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
678 It exists to allow specialized handling on the server side.""" |