Mercurial > hg
annotate hgext/lfs/blobstore.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | 698667eb7523 |
children | 687b865b95ad |
rev | line source |
---|---|
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
9 |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
10 import contextlib |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
11 import errno |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
12 import hashlib |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
14 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
15 import re |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
16 import socket |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
17 |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
18 from mercurial.i18n import _ |
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
19 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
20 from mercurial import ( |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
21 encoding, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
22 error, |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
23 node, |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
24 pathutil, |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
25 pycompat, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
26 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
27 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 vfs as vfsmod, |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
29 worker, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
30 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
31 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
32 from mercurial.utils import stringutil |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
33 |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
34 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
35 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
36 # 64 bytes for SHA256 |
36455
9e3cb58c7ab3
py3: make sure regexes are bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35927
diff
changeset
|
37 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
38 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
39 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
40 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
41 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
42 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
43 if not _lfsre.match(path): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
44 raise error.ProgrammingError(b'unexpected lfs path: %s' % path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
45 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
46 |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
47 def walk(self, path=None, onerror=None): |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
48 """Yield (dirpath, [], oids) tuple for blobs under path |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
49 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
50 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
51 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
52 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
53 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
54 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
55 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
56 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
57 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
58 for dirpath, dirs, files in os.walk( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
59 self.reljoin(self.base, path or b''), onerror=onerror |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
60 ): |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
61 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
62 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
63 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
64 if len(dirpath) == 2: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
65 oids.extend( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
66 [dirpath + f for f in files if _lfsre.match(dirpath + f)] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
67 ) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
68 |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
69 yield ('', [], oids) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
70 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
71 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
72 class nullvfs(lfsvfs): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
73 def __init__(self): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
74 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
75 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
76 def exists(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
77 return False |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
78 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
79 def read(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
80 # store.read() calls into here if the blob doesn't exist in its |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
81 # self.vfs. Raise the same error as a normal vfs when asked to read a |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
82 # file that doesn't exist. The only difference is the full file path |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
83 # isn't available in the error. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
84 raise IOError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
85 errno.ENOENT, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
86 pycompat.sysstr(b'%s: No such file or directory' % oid), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
87 ) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
88 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
89 def walk(self, path=None, onerror=None): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
90 return (b'', [], []) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
91 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
92 def write(self, oid, data): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
93 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
94 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
95 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
96 class filewithprogress(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
97 """a file-like object that supports __len__ and read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
98 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
99 Useful to provide progress information for how many bytes are read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
100 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
101 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
102 def __init__(self, fp, callback): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
103 self._fp = fp |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
104 self._callback = callback # func(readsize) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
105 fp.seek(0, os.SEEK_END) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
106 self._len = fp.tell() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
107 fp.seek(0) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
108 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
109 def __len__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
110 return self._len |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
111 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 def read(self, size): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 if self._fp is None: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
114 return b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
115 data = self._fp.read(size) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
116 if data: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
117 if self._callback: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
118 self._callback(len(data)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
119 else: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
120 self._fp.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
121 self._fp = None |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
122 return data |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
123 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
124 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
125 class local(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
126 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
127 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
128 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
129 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
130 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
131 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
132 def __init__(self, repo): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
133 fullpath = repo.svfs.join(b'lfs/objects') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
134 self.vfs = lfsvfs(fullpath) |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
135 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
136 if repo.ui.configbool(b'experimental', b'lfs.disableusercache'): |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
137 self.cachevfs = nullvfs() |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
138 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
139 usercache = lfutil._usercachedir(repo.ui, b'lfs') |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
140 self.cachevfs = lfsvfs(usercache) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
141 self.ui = repo.ui |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
142 |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
143 def open(self, oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
144 """Open a read-only file descriptor to the named blob, in either the |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
145 usercache or the local store.""" |
35537
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
146 # The usercache is the most likely place to hold the file. Commit will |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
147 # write to both it and the local store, as will anything that downloads |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
148 # the blobs. However, things like clone without an update won't |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
149 # populate the local store. For an init + push of a local clone, |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
150 # the usercache is the only place it _could_ be. If not present, the |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
151 # missing file msg here will indicate the local repo, not the usercache. |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
152 if self.cachevfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
153 return self.cachevfs(oid, b'rb') |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
154 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
155 return self.vfs(oid, b'rb') |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
156 |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
157 def download(self, oid, src): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
158 """Read the blob from the remote source in chunks, verify the content, |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
159 and write to this local blobstore.""" |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
160 sha256 = hashlib.sha256() |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
161 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
162 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
163 for chunk in util.filechunkiter(src, size=1048576): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
164 fp.write(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
165 sha256.update(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
166 |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
167 realoid = node.hex(sha256.digest()) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
168 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
169 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
170 _(b'corrupt remote lfs object: %s') % oid |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
171 ) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
172 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
173 self._linktousercache(oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
174 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
175 def write(self, oid, data): |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
176 """Write blob to local blobstore. |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
177 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
178 This should only be called from the filelog during a commit or similar. |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
179 As such, there is no need to verify the data. Imports from a remote |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
180 store must use ``download()`` instead.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
181 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
182 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
183 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
184 self._linktousercache(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
185 |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
186 def linkfromusercache(self, oid): |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
187 """Link blobs found in the user cache into this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
188 |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
189 The server module needs to do this when it lets the client know not to |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
190 upload the blob, to ensure it is always available in this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
191 Normally this is done implicitly when the client reads or writes the |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
192 blob, but that doesn't happen when the server tells the client that it |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
193 already has the blob. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
194 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
195 if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
196 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
197 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
198 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
199 def _linktousercache(self, oid): |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
200 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
201 # the local store on success, but truncate, write and link on failure? |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
202 if not self.cachevfs.exists(oid) and not isinstance( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
203 self.cachevfs, nullvfs |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
204 ): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
205 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
206 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
207 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
208 def read(self, oid, verify=True): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
209 """Read blob from local blobstore.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
210 if not self.vfs.exists(oid): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
211 blob = self._read(self.cachevfs, oid, verify) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
212 |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
213 # Even if revlog will verify the content, it needs to be verified |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
214 # now before making the hardlink to avoid propagating corrupt blobs. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
215 # Don't abort if corruption is detected, because `hg verify` will |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
216 # give more useful info about the corruption- simply don't add the |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
217 # hardlink. |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
218 if verify or node.hex(hashlib.sha256(blob).digest()) == oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
219 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
220 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
221 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
222 self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
223 blob = self._read(self.vfs, oid, verify) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
224 return blob |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
225 |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
226 def _read(self, vfs, oid, verify): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
227 """Read blob (after verifying) from the given store""" |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
228 blob = vfs.read(oid) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
229 if verify: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
230 _verify(oid, blob) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
231 return blob |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
232 |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
233 def verify(self, oid): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
234 """Indicate whether or not the hash of the underlying file matches its |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
235 name.""" |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
236 sha256 = hashlib.sha256() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
237 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
238 with self.open(oid) as fp: |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
239 for chunk in util.filechunkiter(fp, size=1048576): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
240 sha256.update(chunk) |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
241 |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
242 return oid == node.hex(sha256.digest()) |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
243 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
244 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
245 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
246 False otherwise.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
247 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
248 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
249 |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
250 def _urlerrorreason(urlerror): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
251 '''Create a friendly message for the given URLError to be used in an |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
252 LfsRemoteError message. |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
253 ''' |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
254 inst = urlerror |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
255 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
256 if isinstance(urlerror.reason, Exception): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
257 inst = urlerror.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
258 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
259 if util.safehasattr(inst, 'reason'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
260 try: # usually it is in the form (errno, strerror) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
261 reason = inst.reason.args[1] |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
262 except (AttributeError, IndexError): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
263 # it might be anything, for example a string |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
264 reason = inst.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
265 if isinstance(reason, pycompat.unicode): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
266 # SSLError of Python 2.7.9 contains a unicode |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
267 reason = encoding.unitolocal(reason) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
268 return reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
269 elif getattr(inst, "strerror", None): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
270 return encoding.strtolocal(inst.strerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
271 else: |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
272 return stringutil.forcebytestr(urlerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
273 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
274 |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
275 class lfsauthhandler(util.urlreq.basehandler): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
276 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
277 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
278 def http_error_401(self, req, fp, code, msg, headers): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
279 """Enforces that any authentication performed is HTTP Basic |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
280 Authentication. No authentication is also acceptable. |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
281 """ |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
282 authreq = headers.get(r'www-authenticate', None) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
283 if authreq: |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
284 scheme = authreq.split()[0] |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
285 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
286 if scheme.lower() != r'basic': |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
287 msg = _(b'the server must support Basic Authentication') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
288 raise util.urlerr.httperror( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
289 req.get_full_url(), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
290 code, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
291 encoding.strfromlocal(msg), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
292 headers, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
293 fp, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
294 ) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
295 return None |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
296 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
297 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
298 class _gitlfsremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
299 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
300 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
301 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
302 baseurl, authinfo = url.authinfo() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
303 self.baseurl = baseurl.rstrip(b'/') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
304 useragent = repo.ui.config(b'experimental', b'lfs.user-agent') |
35440
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
305 if not useragent: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
306 useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
35439
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35433
diff
changeset
|
307 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
308 self.urlopener.add_handler(lfsauthhandler()) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
309 self.retry = ui.configint(b'lfs', b'retry') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
310 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
311 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
312 """Batch upload from local to remote blobstore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
313 self._batch(_deduplicate(pointers), fromstore, b'upload') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
314 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
315 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
316 """Batch download from remote to local blostore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
317 self._batch(_deduplicate(pointers), tostore, b'download') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
318 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
319 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
320 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
321 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
322 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
323 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
324 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
325 objects = [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
326 {r'oid': pycompat.strurl(p.oid()), r'size': p.size()} |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
327 for p in pointers |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
328 ] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
329 requestdata = pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
330 json.dumps( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
331 {r'objects': objects, r'operation': pycompat.strurl(action),} |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
332 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
333 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
334 url = b'%s/objects/batch' % self.baseurl |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
335 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
336 batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
337 batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
338 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
339 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
340 rawjson = rsp.read() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
341 except util.urlerr.httperror as ex: |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
342 hints = { |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
343 400: _( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
344 b'check that lfs serving is enabled on %s and "%s" is ' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
345 b'supported' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
346 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
347 % (self.baseurl, action), |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
348 404: _(b'the "lfs.url" config may be used to override %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
349 % self.baseurl, |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
350 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
351 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
352 raise LfsRemoteError( |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
353 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
354 hint=hint, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
355 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
356 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
357 hint = ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
358 _(b'the "lfs.url" config may be used to override %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
359 % self.baseurl |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
360 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
361 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
362 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
363 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
364 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
365 response = json.loads(rawjson) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
366 except ValueError: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
367 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
368 _(b'LFS server returns invalid JSON: %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
369 % rawjson.encode("utf-8") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
370 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
371 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
372 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
373 self.ui.debug(b'Status: %d\n' % rsp.status) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
374 # lfs-test-server and hg serve return headers in different order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
375 headers = pycompat.bytestr(rsp.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
376 self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
377 |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
378 if r'objects' in response: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
379 response[r'objects'] = sorted( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
380 response[r'objects'], key=lambda p: p[r'oid'] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
381 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
382 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
383 b'%s\n' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
384 % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
385 json.dumps( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
386 response, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
387 indent=2, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
388 separators=(r'', r': '), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
389 sort_keys=True, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
390 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
391 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
392 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
393 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
394 def encodestr(x): |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
395 if isinstance(x, pycompat.unicode): |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
396 return x.encode(u'utf-8') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
397 return x |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
398 |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
399 return pycompat.rapply(encodestr, response) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
400 |
35666 | 401 def _checkforservererror(self, pointers, responses, action): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
402 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
403 |
35694
8a23082f4d93
lfs: correct documentation typo
Matt Harbison <matt_harbison@yahoo.com>
parents:
35666
diff
changeset
|
404 Raises LfsRemoteError if any objects have an error""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
405 for response in responses: |
35666 | 406 # The server should return 404 when objects cannot be found. Some |
407 # server implementation (ex. lfs-test-server) does not set "error" | |
408 # but just removes "download" from "actions". Treat that case | |
409 # as the same as 404 error. | |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
410 if b'error' not in response: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
411 if action == b'download' and action not in response.get( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
412 b'actions', [] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
413 ): |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
414 code = 404 |
35695
dd672e3d059f
lfs: raise an error if the server sends an unsolicited oid
Matt Harbison <matt_harbison@yahoo.com>
parents:
35694
diff
changeset
|
415 else: |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
416 continue |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
417 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
418 # An error dict without a code doesn't make much sense, so |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
419 # treat as a server error. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
420 code = response.get(b'error').get(b'code', 500) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
421 |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
422 ptrmap = {p.oid(): p for p in pointers} |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
423 p = ptrmap.get(response[b'oid'], None) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
424 if p: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
425 filename = getattr(p, 'filename', b'unknown') |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
426 errors = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
427 404: b'The object does not exist', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
428 410: b'The object was removed by the owner', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
429 422: b'Validation error', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
430 500: b'Internal server error', |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
431 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
432 msg = errors.get(code, b'status code %d' % code) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
433 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
434 _(b'LFS server error for "%s": %s') % (filename, msg) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
435 ) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
436 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
437 raise LfsRemoteError( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
438 _(b'LFS server error. Unsolicited response for oid %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
439 % response[b'oid'] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
440 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
441 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
442 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
443 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
444 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
445 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
446 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
447 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
448 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
449 # Scan errors from objects - fail early |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
450 objects = response.get(b'objects', []) |
35666 | 451 self._checkforservererror(pointers, objects, action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
452 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
453 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
454 # objects which exist in the server. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
455 filteredobjects = [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
456 o for o in objects if action in o.get(b'actions', []) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
457 ] |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
458 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
459 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
460 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
461 def _basictransfer(self, obj, action, localstore): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
462 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
463 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
464 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
465 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
466 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
467 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
468 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
469 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
470 """ |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
471 oid = obj[b'oid'] |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
472 href = obj[b'actions'][action].get(b'href') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
473 headers = obj[b'actions'][action].get(b'header', {}).items() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
474 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
475 request = util.urlreq.request(pycompat.strurl(href)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
476 if action == b'upload': |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
477 # If uploading blobs, read data from local blobstore. |
37217
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
478 if not localstore.verify(oid): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
479 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
480 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
481 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
482 ) |
35526
e8f80529abeb
lfs: use the local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35525
diff
changeset
|
483 request.data = filewithprogress(localstore.open(oid), None) |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
484 request.get_method = lambda: r'PUT' |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
485 request.add_header(r'Content-Type', r'application/octet-stream') |
41440
1bc01490178a
lfs: explicitly add the Content-Length header when uploading blobs, for py3
Matt Harbison <matt_harbison@yahoo.com>
parents:
41431
diff
changeset
|
486 request.add_header(r'Content-Length', len(request.data)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
487 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
488 for k, v in headers: |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
489 request.add_header(pycompat.strurl(k), pycompat.strurl(v)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
490 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
491 response = b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
492 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
493 with contextlib.closing(self.urlopener.open(request)) as req: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
494 ui = self.ui # Shorten debug lines |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
495 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
496 ui.debug(b'Status: %d\n' % req.status) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
497 # lfs-test-server and hg serve return headers in different |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
498 # order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
499 headers = pycompat.bytestr(req.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
500 ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
501 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
502 if action == b'download': |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
503 # If downloading blobs, store downloaded data to local |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
504 # blobstore |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
505 localstore.download(oid, req) |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
506 else: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
507 while True: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
508 data = req.read(1048576) |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
509 if not data: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
510 break |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
511 response += data |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
512 if response: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
513 ui.debug(b'lfs %s response: %s' % (action, response)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
514 except util.urlerr.httperror as ex: |
35734
b4e1d0654736
lfs: dump the full response on httperror in debug mode
Matt Harbison <matt_harbison@yahoo.com>
parents:
35733
diff
changeset
|
515 if self.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
516 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
517 b'%s: %s\n' % (oid, ex.read()) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
518 ) # XXX: also bytes? |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
519 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
520 _(b'LFS HTTP error: %s (oid=%s, action=%s)') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
521 % (stringutil.forcebytestr(ex), oid, action) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
522 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
523 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
524 hint = _(b'attempted connection to %s') % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
525 util.urllibcompat.getfullurl(request) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
526 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
527 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
528 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
529 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
530 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
531 def _batch(self, pointers, localstore, action): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
532 if action not in [b'upload', b'download']: |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
533 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
534 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
535 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
536 objects = self._extractobjects(response, pointers, action) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
537 total = sum(x.get(b'size', 0) for x in objects) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
538 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
539 for obj in objects: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
540 sizes[obj.get(b'oid')] = obj.get(b'size', 0) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
541 topic = { |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
542 b'upload': _(b'lfs uploading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
543 b'download': _(b'lfs downloading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
544 }[action] |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
545 if len(objects) > 1: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
546 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
547 _(b'lfs: need to transfer %d objects (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
548 % (len(objects), util.bytecount(total)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
549 ) |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
550 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
551 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
552 for obj in chunk: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
553 objsize = obj.get(b'size', 0) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
554 if self.ui.verbose: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
555 if action == b'download': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
556 msg = _(b'lfs: downloading %s (%s)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
557 elif action == b'upload': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
558 msg = _(b'lfs: uploading %s (%s)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
559 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
560 msg % (obj.get(b'oid'), util.bytecount(objsize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
561 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
562 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
563 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
564 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
565 self._basictransfer(obj, action, localstore) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
566 yield 1, obj.get(b'oid') |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
567 break |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
568 except socket.error as ex: |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
569 if retry > 0: |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
570 self.ui.note( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
571 _(b'lfs: failed: %r (remaining retry %d)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
572 % (stringutil.forcebytestr(ex), retry) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
573 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
574 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
575 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
576 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
577 |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
578 # Until https multiplexing gets sorted out |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
579 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
580 oids = worker.worker( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
581 self.ui, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
582 0.1, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
583 transfer, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
584 (), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
585 sorted(objects, key=lambda o: o.get(b'oid')), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
586 ) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
587 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
588 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
589 |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
590 with self.ui.makeprogress(topic, total=total) as progress: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
591 progress.update(0) |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
592 processed = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
593 blobs = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
594 for _one, oid in oids: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
595 processed += sizes[oid] |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
596 blobs += 1 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
597 progress.update(processed) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
598 self.ui.note(_(b'lfs: processed: %s\n') % oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
599 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
600 if blobs > 0: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
601 if action == b'upload': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
602 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
603 _(b'lfs: uploaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
604 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
605 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
606 elif action == b'download': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
607 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
608 _(b'lfs: downloaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
609 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
610 ) |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
611 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
612 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
613 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
614 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
615 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
616 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
617 h.close() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
618 getattr(h, "close_all", lambda: None)() |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
619 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
620 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
621 class _dummyremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
622 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
623 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
624 def __init__(self, repo, url): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
625 fullpath = repo.vfs.join(b'lfs', url.path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
626 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
627 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
628 def writebatch(self, pointers, fromstore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
629 for p in _deduplicate(pointers): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
630 content = fromstore.read(p.oid(), verify=True) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
631 with self.vfs(p.oid(), b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
632 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
633 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
634 def readbatch(self, pointers, tostore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
635 for p in _deduplicate(pointers): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
636 with self.vfs(p.oid(), b'rb') as fp: |
35552
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
637 tostore.download(p.oid(), fp) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
638 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
639 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
640 class _nullremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
641 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
642 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
643 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
644 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
645 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
646 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
647 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
648 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
649 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
650 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
651 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
652 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
653 class _promptremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
654 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
655 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
656 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
657 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
658 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
659 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
660 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
661 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
662 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
663 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
664 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
665 def _prompt(self): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
666 raise error.Abort(_(b'lfs.url needs to be configured')) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
667 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
668 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
669 _storemap = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
670 b'https': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
671 b'http': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
672 b'file': _dummyremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
673 b'null': _nullremote, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
674 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
675 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
676 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
677 |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
678 def _deduplicate(pointers): |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
679 """Remove any duplicate oids that exist in the list""" |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
680 reduced = util.sortdict() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
681 for p in pointers: |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
682 reduced[p.oid()] = p |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
683 return reduced.values() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
684 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
685 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
686 def _verify(oid, content): |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
687 realoid = node.hex(hashlib.sha256(content).digest()) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
688 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
689 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
690 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
691 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
692 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
693 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
694 |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
695 def remote(repo, remote=None): |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
696 """remotestore factory. return a store in _storemap depending on config |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
697 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
698 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
699 infer the endpoint, based on the remote repository using the same path |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
700 adjustments as git. As an extension, 'http' is supported as well so that |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
701 ``hg serve`` works out of the box. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
702 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
703 https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
704 """ |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
705 lfsurl = repo.ui.config(b'lfs', b'url') |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
706 url = util.url(lfsurl or '') |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
707 if lfsurl is None: |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
708 if remote: |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
709 path = remote |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
710 elif util.safehasattr(repo, '_subtoppath'): |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
711 # The pull command sets this during the optional update phase, which |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
712 # tells exactly where the pull originated, whether 'paths.default' |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
713 # or explicit. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
714 path = repo._subtoppath |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
715 else: |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
716 # TODO: investigate 'paths.remote:lfsurl' style path customization, |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
717 # and fall back to inferring from 'paths.remote' if unspecified. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
718 path = repo.ui.config(b'paths', b'default') or b'' |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
719 |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
720 defaulturl = util.url(path) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
721 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
722 # TODO: support local paths as well. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
723 # TODO: consider the ssh -> https transformation that git applies |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
724 if defaulturl.scheme in (b'http', b'https'): |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
725 if defaulturl.path and defaulturl.path[:-1] != b'/': |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
726 defaulturl.path += b'/' |
37691
d241e6632669
lfs: fix the inferred remote store path when using a --prefix
Matt Harbison <matt_harbison@yahoo.com>
parents:
37565
diff
changeset
|
727 defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
728 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
729 url = util.url(bytes(defaulturl)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
730 repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
731 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
732 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
733 if scheme not in _storemap: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
734 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
735 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
736 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
737 |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39457
diff
changeset
|
738 class LfsRemoteError(error.StorageError): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
739 pass |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
740 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
741 |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
742 class LfsCorruptionError(error.Abort): |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
743 """Raised when a corrupt blob is detected, aborting an operation |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
744 |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
745 It exists to allow specialized handling on the server side.""" |