Mercurial > hg
annotate hgext/lfs/blobstore.py @ 43105:649d3ac37a12
py3: define and use pycompat.iteritems() for hgext/
.iteritems() -> .items() is the last source transform being performed.
But it is also the most widely used.
This commit adds a pycompat.iteritems symbol and imports it in place
of .iteritems() for usage in hgext/. I chose to stop at just hgext/
because the patch will be large and it is an easy boundary to stop at
since we can disable source transformation on a per-package basis.
There are places where the type does implement items() and we could
call items() directly. However, this would require critical thought
and I thought it would be easier to just blindly change the code. We
know which call sites need to be audited in the future because they
have "pycompat.iteritems."
With this change, we no longer perform source transformation on
hgext!
Differential Revision: https://phab.mercurial-scm.org/D7014
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Sun, 06 Oct 2019 19:25:18 -0400 |
parents | 127cc1f72e70 |
children | 579672b347d2 |
rev | line source |
---|---|
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
9 |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
10 import contextlib |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
11 import errno |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
12 import hashlib |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
14 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
15 import re |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
16 import socket |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
17 |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
18 from mercurial.i18n import _ |
43089
c59eb1560c44
py3: manually import getattr where it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43077
diff
changeset
|
19 from mercurial.pycompat import getattr |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
20 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
21 from mercurial import ( |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
22 encoding, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
23 error, |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
24 node, |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
25 pathutil, |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
26 pycompat, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
27 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
29 vfs as vfsmod, |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
30 worker, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
31 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
32 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
33 from mercurial.utils import stringutil |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
34 |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
35 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
36 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
37 # 64 bytes for SHA256 |
36455
9e3cb58c7ab3
py3: make sure regexes are bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35927
diff
changeset
|
38 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
39 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
40 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
41 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
42 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
43 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
44 if not _lfsre.match(path): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
45 raise error.ProgrammingError(b'unexpected lfs path: %s' % path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
46 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
47 |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
48 def walk(self, path=None, onerror=None): |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
49 """Yield (dirpath, [], oids) tuple for blobs under path |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
50 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
51 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
52 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
53 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
54 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
55 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
56 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
57 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
58 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
59 for dirpath, dirs, files in os.walk( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
60 self.reljoin(self.base, path or b''), onerror=onerror |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
61 ): |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
62 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
63 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
64 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
65 if len(dirpath) == 2: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
66 oids.extend( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
67 [dirpath + f for f in files if _lfsre.match(dirpath + f)] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
68 ) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
69 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
70 yield (b'', [], oids) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
71 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
72 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
73 class nullvfs(lfsvfs): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
74 def __init__(self): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
75 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
76 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
77 def exists(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
78 return False |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
79 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
80 def read(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
81 # store.read() calls into here if the blob doesn't exist in its |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
82 # self.vfs. Raise the same error as a normal vfs when asked to read a |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
83 # file that doesn't exist. The only difference is the full file path |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
84 # isn't available in the error. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
85 raise IOError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
86 errno.ENOENT, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
87 pycompat.sysstr(b'%s: No such file or directory' % oid), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
88 ) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
89 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
90 def walk(self, path=None, onerror=None): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
91 return (b'', [], []) |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
92 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
93 def write(self, oid, data): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
94 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
95 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
96 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
97 class filewithprogress(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
98 """a file-like object that supports __len__ and read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
99 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
100 Useful to provide progress information for how many bytes are read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
101 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
102 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
103 def __init__(self, fp, callback): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
104 self._fp = fp |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
105 self._callback = callback # func(readsize) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
106 fp.seek(0, os.SEEK_END) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
107 self._len = fp.tell() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
108 fp.seek(0) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
109 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
110 def __len__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
111 return self._len |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 def read(self, size): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
114 if self._fp is None: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
115 return b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
116 data = self._fp.read(size) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
117 if data: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
118 if self._callback: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
119 self._callback(len(data)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
120 else: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
121 self._fp.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
122 self._fp = None |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
123 return data |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
124 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
125 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
126 class local(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
127 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
128 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
129 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
130 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
131 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
132 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
133 def __init__(self, repo): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
134 fullpath = repo.svfs.join(b'lfs/objects') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
135 self.vfs = lfsvfs(fullpath) |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
136 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
137 if repo.ui.configbool(b'experimental', b'lfs.disableusercache'): |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
138 self.cachevfs = nullvfs() |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
139 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
140 usercache = lfutil._usercachedir(repo.ui, b'lfs') |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
141 self.cachevfs = lfsvfs(usercache) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
142 self.ui = repo.ui |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
143 |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
144 def open(self, oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
145 """Open a read-only file descriptor to the named blob, in either the |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
146 usercache or the local store.""" |
35537
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
147 # The usercache is the most likely place to hold the file. Commit will |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
148 # write to both it and the local store, as will anything that downloads |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
149 # the blobs. However, things like clone without an update won't |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
150 # populate the local store. For an init + push of a local clone, |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
151 # the usercache is the only place it _could_ be. If not present, the |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
152 # missing file msg here will indicate the local repo, not the usercache. |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
153 if self.cachevfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
154 return self.cachevfs(oid, b'rb') |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
155 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
156 return self.vfs(oid, b'rb') |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
157 |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
158 def download(self, oid, src): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
159 """Read the blob from the remote source in chunks, verify the content, |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
160 and write to this local blobstore.""" |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
161 sha256 = hashlib.sha256() |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
162 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
163 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
164 for chunk in util.filechunkiter(src, size=1048576): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
165 fp.write(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
166 sha256.update(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
167 |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
168 realoid = node.hex(sha256.digest()) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
169 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
170 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
171 _(b'corrupt remote lfs object: %s') % oid |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
172 ) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
173 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
174 self._linktousercache(oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
175 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
176 def write(self, oid, data): |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
177 """Write blob to local blobstore. |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
178 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
179 This should only be called from the filelog during a commit or similar. |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
180 As such, there is no need to verify the data. Imports from a remote |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
181 store must use ``download()`` instead.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
182 with self.vfs(oid, b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
183 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
184 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
185 self._linktousercache(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
186 |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
187 def linkfromusercache(self, oid): |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
188 """Link blobs found in the user cache into this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
189 |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
190 The server module needs to do this when it lets the client know not to |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
191 upload the blob, to ensure it is always available in this store. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
192 Normally this is done implicitly when the client reads or writes the |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
193 blob, but that doesn't happen when the server tells the client that it |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
194 already has the blob. |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
195 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
196 if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
197 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
39457
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
198 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
a913d2892e17
lfs: ensure the blob is linked to the remote store on skipped uploads
Matt Harbison <matt_harbison@yahoo.com>
parents:
39389
diff
changeset
|
199 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
200 def _linktousercache(self, oid): |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
201 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
202 # the local store on success, but truncate, write and link on failure? |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
203 if not self.cachevfs.exists(oid) and not isinstance( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
204 self.cachevfs, nullvfs |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
205 ): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
206 self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
207 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
208 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
209 def read(self, oid, verify=True): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
210 """Read blob from local blobstore.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
211 if not self.vfs.exists(oid): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
212 blob = self._read(self.cachevfs, oid, verify) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
213 |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
214 # Even if revlog will verify the content, it needs to be verified |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
215 # now before making the hardlink to avoid propagating corrupt blobs. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
216 # Don't abort if corruption is detected, because `hg verify` will |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
217 # give more useful info about the corruption- simply don't add the |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
218 # hardlink. |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
219 if verify or node.hex(hashlib.sha256(blob).digest()) == oid: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
220 self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
221 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
222 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
223 self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
224 blob = self._read(self.vfs, oid, verify) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
225 return blob |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
226 |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
227 def _read(self, vfs, oid, verify): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
228 """Read blob (after verifying) from the given store""" |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
229 blob = vfs.read(oid) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
230 if verify: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
231 _verify(oid, blob) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
232 return blob |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
233 |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
234 def verify(self, oid): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
235 """Indicate whether or not the hash of the underlying file matches its |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
236 name.""" |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
237 sha256 = hashlib.sha256() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
238 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
239 with self.open(oid) as fp: |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
240 for chunk in util.filechunkiter(fp, size=1048576): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
241 sha256.update(chunk) |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
242 |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
243 return oid == node.hex(sha256.digest()) |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
244 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
245 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
246 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
247 False otherwise.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
248 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
249 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
250 |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
251 def _urlerrorreason(urlerror): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
252 '''Create a friendly message for the given URLError to be used in an |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
253 LfsRemoteError message. |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
254 ''' |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
255 inst = urlerror |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
256 |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
257 if isinstance(urlerror.reason, Exception): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
258 inst = urlerror.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
259 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
260 if util.safehasattr(inst, b'reason'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
261 try: # usually it is in the form (errno, strerror) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
262 reason = inst.reason.args[1] |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
263 except (AttributeError, IndexError): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
264 # it might be anything, for example a string |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
265 reason = inst.reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
266 if isinstance(reason, pycompat.unicode): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
267 # SSLError of Python 2.7.9 contains a unicode |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
268 reason = encoding.unitolocal(reason) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
269 return reason |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
270 elif getattr(inst, "strerror", None): |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
271 return encoding.strtolocal(inst.strerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
272 else: |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
273 return stringutil.forcebytestr(urlerror) |
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
274 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
275 |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
276 class lfsauthhandler(util.urlreq.basehandler): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
277 handler_order = 480 # Before HTTPDigestAuthHandler (== 490) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
278 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
279 def http_error_401(self, req, fp, code, msg, headers): |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
280 """Enforces that any authentication performed is HTTP Basic |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
281 Authentication. No authentication is also acceptable. |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
282 """ |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
283 authreq = headers.get(r'www-authenticate', None) |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
284 if authreq: |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
285 scheme = authreq.split()[0] |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
286 |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
287 if scheme.lower() != r'basic': |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
288 msg = _(b'the server must support Basic Authentication') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
289 raise util.urlerr.httperror( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
290 req.get_full_url(), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
291 code, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
292 encoding.strfromlocal(msg), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
293 headers, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
294 fp, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
295 ) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
296 return None |
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
297 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
298 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
299 class _gitlfsremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
300 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
301 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
302 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
303 baseurl, authinfo = url.authinfo() |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
304 self.baseurl = baseurl.rstrip(b'/') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
305 useragent = repo.ui.config(b'experimental', b'lfs.user-agent') |
35440
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
306 if not useragent: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
307 useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
35439
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35433
diff
changeset
|
308 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
41607
698667eb7523
lfs: disable all authentication except Basic for HTTP(S) connections
Matt Harbison <matt_harbison@yahoo.com>
parents:
41440
diff
changeset
|
309 self.urlopener.add_handler(lfsauthhandler()) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
310 self.retry = ui.configint(b'lfs', b'retry') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
311 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
312 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
313 """Batch upload from local to remote blobstore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
314 self._batch(_deduplicate(pointers), fromstore, b'upload') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
315 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
316 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
317 """Batch download from remote to local blostore.""" |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
318 self._batch(_deduplicate(pointers), tostore, b'download') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
319 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
320 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
321 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
322 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
323 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
324 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
325 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
326 objects = [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
327 {r'oid': pycompat.strurl(p.oid()), r'size': p.size()} |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
328 for p in pointers |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
329 ] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
330 requestdata = pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
331 json.dumps( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
332 {r'objects': objects, r'operation': pycompat.strurl(action),} |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
333 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
334 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
335 url = b'%s/objects/batch' % self.baseurl |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
336 batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
337 batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
338 batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
339 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
340 with contextlib.closing(self.urlopener.open(batchreq)) as rsp: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
341 rawjson = rsp.read() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
342 except util.urlerr.httperror as ex: |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
343 hints = { |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
344 400: _( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
345 b'check that lfs serving is enabled on %s and "%s" is ' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
346 b'supported' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
347 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
348 % (self.baseurl, action), |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
349 404: _(b'the "lfs.url" config may be used to override %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
350 % self.baseurl, |
40660
9f78d10742af
lfs: improve the hints for common errors in the Batch API
Matt Harbison <matt_harbison@yahoo.com>
parents:
40659
diff
changeset
|
351 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
352 hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
353 raise LfsRemoteError( |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
354 _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
355 hint=hint, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
356 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
357 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
358 hint = ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
359 _(b'the "lfs.url" config may be used to override %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
360 % self.baseurl |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
361 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
362 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
363 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
364 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
365 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
366 response = json.loads(rawjson) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
367 except ValueError: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
368 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
369 _(b'LFS server returns invalid JSON: %s') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
370 % rawjson.encode("utf-8") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
371 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
372 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
373 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
374 self.ui.debug(b'Status: %d\n' % rsp.status) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
375 # lfs-test-server and hg serve return headers in different order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
376 headers = pycompat.bytestr(rsp.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
377 self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
378 |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
379 if r'objects' in response: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
380 response[r'objects'] = sorted( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
381 response[r'objects'], key=lambda p: p[r'oid'] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
382 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
383 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
384 b'%s\n' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
385 % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
386 json.dumps( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
387 response, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
388 indent=2, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
389 separators=(r'', r': '), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
390 sort_keys=True, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
391 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
392 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
393 ) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
394 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
395 def encodestr(x): |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
396 if isinstance(x, pycompat.unicode): |
43091
127cc1f72e70
py3: stop normalizing .encode()/.decode() arguments to unicode
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43089
diff
changeset
|
397 return x.encode('utf-8') |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
398 return x |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
399 |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
400 return pycompat.rapply(encodestr, response) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
401 |
35666 | 402 def _checkforservererror(self, pointers, responses, action): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
403 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
404 |
35694
8a23082f4d93
lfs: correct documentation typo
Matt Harbison <matt_harbison@yahoo.com>
parents:
35666
diff
changeset
|
405 Raises LfsRemoteError if any objects have an error""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
406 for response in responses: |
35666 | 407 # The server should return 404 when objects cannot be found. Some |
408 # server implementation (ex. lfs-test-server) does not set "error" | |
409 # but just removes "download" from "actions". Treat that case | |
410 # as the same as 404 error. | |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
411 if b'error' not in response: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
412 if action == b'download' and action not in response.get( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
413 b'actions', [] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
414 ): |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
415 code = 404 |
35695
dd672e3d059f
lfs: raise an error if the server sends an unsolicited oid
Matt Harbison <matt_harbison@yahoo.com>
parents:
35694
diff
changeset
|
416 else: |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
417 continue |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
418 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
419 # An error dict without a code doesn't make much sense, so |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
420 # treat as a server error. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
421 code = response.get(b'error').get(b'code', 500) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
422 |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
423 ptrmap = {p.oid(): p for p in pointers} |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
424 p = ptrmap.get(response[b'oid'], None) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
425 if p: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
426 filename = getattr(p, 'filename', b'unknown') |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
427 errors = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
428 404: b'The object does not exist', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
429 410: b'The object was removed by the owner', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
430 422: b'Validation error', |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
431 500: b'Internal server error', |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
432 } |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
433 msg = errors.get(code, b'status code %d' % code) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
434 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
435 _(b'LFS server error for "%s": %s') % (filename, msg) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
436 ) |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
437 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
438 raise LfsRemoteError( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
439 _(b'LFS server error. Unsolicited response for oid %s') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
440 % response[b'oid'] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
441 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
442 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
443 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
444 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
445 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
446 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
447 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
448 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
449 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
450 # Scan errors from objects - fail early |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
451 objects = response.get(b'objects', []) |
35666 | 452 self._checkforservererror(pointers, objects, action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
453 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
454 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
455 # objects which exist in the server. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
456 filteredobjects = [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
457 o for o in objects if action in o.get(b'actions', []) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
458 ] |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
459 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
460 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
461 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
462 def _basictransfer(self, obj, action, localstore): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
463 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
464 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
465 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
466 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
467 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
468 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
469 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
470 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
471 """ |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
472 oid = obj[b'oid'] |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
473 href = obj[b'actions'][action].get(b'href') |
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
474 headers = obj[b'actions'][action].get(b'header', {}).items() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
475 |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
476 request = util.urlreq.request(pycompat.strurl(href)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
477 if action == b'upload': |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
478 # If uploading blobs, read data from local blobstore. |
37217
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
479 if not localstore.verify(oid): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
480 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
481 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
482 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
483 ) |
35526
e8f80529abeb
lfs: use the local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35525
diff
changeset
|
484 request.data = filewithprogress(localstore.open(oid), None) |
41427
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
485 request.get_method = lambda: r'PUT' |
40efcf78f3df
py3: raw stringify various JSON and HTTP headers in the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
41426
diff
changeset
|
486 request.add_header(r'Content-Type', r'application/octet-stream') |
41440
1bc01490178a
lfs: explicitly add the Content-Length header when uploading blobs, for py3
Matt Harbison <matt_harbison@yahoo.com>
parents:
41431
diff
changeset
|
487 request.add_header(r'Content-Length', len(request.data)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
488 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
489 for k, v in headers: |
41429
7df10ea7a5b8
py3: byteify the decoded JSON responses upon receipt in the LFS blobstore
Matt Harbison <matt_harbison@yahoo.com>
parents:
41427
diff
changeset
|
490 request.add_header(pycompat.strurl(k), pycompat.strurl(v)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
491 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
492 response = b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
493 try: |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
494 with contextlib.closing(self.urlopener.open(request)) as req: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
495 ui = self.ui # Shorten debug lines |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
496 if self.ui.debugflag: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
497 ui.debug(b'Status: %d\n' % req.status) |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
498 # lfs-test-server and hg serve return headers in different |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
499 # order |
41431
0b636d1720a0
lfs: strip the response headers from the Batch API before printing
Matt Harbison <matt_harbison@yahoo.com>
parents:
41429
diff
changeset
|
500 headers = pycompat.bytestr(req.info()).strip() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
501 ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
502 |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
503 if action == b'download': |
40665
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
504 # If downloading blobs, store downloaded data to local |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
505 # blobstore |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
506 localstore.download(oid, req) |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
507 else: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
508 while True: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
509 data = req.read(1048576) |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
510 if not data: |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
511 break |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
512 response += data |
fb379b78b93e
lfs: ensure that the return of urlopener.open() is closed
Matt Harbison <matt_harbison@yahoo.com>
parents:
40662
diff
changeset
|
513 if response: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
514 ui.debug(b'lfs %s response: %s' % (action, response)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
515 except util.urlerr.httperror as ex: |
35734
b4e1d0654736
lfs: dump the full response on httperror in debug mode
Matt Harbison <matt_harbison@yahoo.com>
parents:
35733
diff
changeset
|
516 if self.ui.debugflag: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
517 self.ui.debug( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
518 b'%s: %s\n' % (oid, ex.read()) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
519 ) # XXX: also bytes? |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
520 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
521 _(b'LFS HTTP error: %s (oid=%s, action=%s)') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
522 % (stringutil.forcebytestr(ex), oid, action) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
523 ) |
40661
380f5131ee7b
lfs: handle URLErrors to add additional information
Matt Harbison <matt_harbison@yahoo.com>
parents:
40660
diff
changeset
|
524 except util.urlerr.urlerror as ex: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
525 hint = _(b'attempted connection to %s') % pycompat.bytesurl( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
526 util.urllibcompat.getfullurl(request) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
527 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
528 raise LfsRemoteError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
529 _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
530 ) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
531 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
532 def _batch(self, pointers, localstore, action): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
533 if action not in [b'upload', b'download']: |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
534 raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
535 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
536 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
537 objects = self._extractobjects(response, pointers, action) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
538 total = sum(x.get(b'size', 0) for x in objects) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
539 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
540 for obj in objects: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
541 sizes[obj.get(b'oid')] = obj.get(b'size', 0) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
542 topic = { |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
543 b'upload': _(b'lfs uploading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
544 b'download': _(b'lfs downloading'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
545 }[action] |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
546 if len(objects) > 1: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
547 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
548 _(b'lfs: need to transfer %d objects (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
549 % (len(objects), util.bytecount(total)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
550 ) |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
551 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
552 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
553 for obj in chunk: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
554 objsize = obj.get(b'size', 0) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
555 if self.ui.verbose: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
556 if action == b'download': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
557 msg = _(b'lfs: downloading %s (%s)\n') |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
558 elif action == b'upload': |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
559 msg = _(b'lfs: uploading %s (%s)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
560 self.ui.note( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
561 msg % (obj.get(b'oid'), util.bytecount(objsize)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
562 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
563 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
564 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
565 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
566 self._basictransfer(obj, action, localstore) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
567 yield 1, obj.get(b'oid') |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
568 break |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
569 except socket.error as ex: |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
570 if retry > 0: |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
571 self.ui.note( |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
572 _(b'lfs: failed: %r (remaining retry %d)\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
573 % (stringutil.forcebytestr(ex), retry) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
574 ) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
575 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
576 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
577 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
578 |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
579 # Until https multiplexing gets sorted out |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
580 if self.ui.configbool(b'experimental', b'lfs.worker-enable'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
581 oids = worker.worker( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
582 self.ui, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
583 0.1, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
584 transfer, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
585 (), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
586 sorted(objects, key=lambda o: o.get(b'oid')), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
587 ) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
588 else: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
589 oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
590 |
39389
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
591 with self.ui.makeprogress(topic, total=total) as progress: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
592 progress.update(0) |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
593 processed = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
594 blobs = 0 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
595 for _one, oid in oids: |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
596 processed += sizes[oid] |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
597 blobs += 1 |
b26350d9d7b5
lfs: use a context manager to control the progress bar lifetime
Matt Harbison <matt_harbison@yahoo.com>
parents:
38405
diff
changeset
|
598 progress.update(processed) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
599 self.ui.note(_(b'lfs: processed: %s\n') % oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
600 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
601 if blobs > 0: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
602 if action == b'upload': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
603 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
604 _(b'lfs: uploaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
605 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
606 ) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
607 elif action == b'download': |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
608 self.ui.status( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
609 _(b'lfs: downloaded %d files (%s)\n') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
610 % (blobs, util.bytecount(processed)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
611 ) |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
612 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
613 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
614 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
615 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
616 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
617 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
618 h.close() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
619 getattr(h, "close_all", lambda: None)() |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
620 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
621 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
622 class _dummyremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
623 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
624 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
625 def __init__(self, repo, url): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
626 fullpath = repo.vfs.join(b'lfs', url.path) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
627 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
628 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
629 def writebatch(self, pointers, fromstore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
630 for p in _deduplicate(pointers): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
631 content = fromstore.read(p.oid(), verify=True) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
632 with self.vfs(p.oid(), b'wb', atomictemp=True) as fp: |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
633 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
634 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
635 def readbatch(self, pointers, tostore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
636 for p in _deduplicate(pointers): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
637 with self.vfs(p.oid(), b'rb') as fp: |
35552
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
638 tostore.download(p.oid(), fp) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
639 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
640 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
641 class _nullremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
642 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
643 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
644 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
645 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
646 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
647 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
648 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
649 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
650 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
651 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
652 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
653 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
654 class _promptremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
655 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
656 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
657 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
658 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
659 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
660 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
661 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
662 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
663 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
664 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
665 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
666 def _prompt(self): |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
667 raise error.Abort(_(b'lfs.url needs to be configured')) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
668 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
669 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
670 _storemap = { |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
671 b'https': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
672 b'http': _gitlfsremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
673 b'file': _dummyremote, |
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
674 b'null': _nullremote, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
675 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
676 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
677 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
678 |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
679 def _deduplicate(pointers): |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
680 """Remove any duplicate oids that exist in the list""" |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
681 reduced = util.sortdict() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
682 for p in pointers: |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
683 reduced[p.oid()] = p |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
684 return reduced.values() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
685 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
686 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
687 def _verify(oid, content): |
40675
9fcf8084ada8
py3: use node.hex(m.digest()) instead of m.hexdigest()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40665
diff
changeset
|
688 realoid = node.hex(hashlib.sha256(content).digest()) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
689 if realoid != oid: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
690 raise LfsCorruptionError( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
691 _(b'detected corrupt lfs object: %s') % oid, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
692 hint=_(b'run hg verify'), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
693 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
694 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
695 |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
696 def remote(repo, remote=None): |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
697 """remotestore factory. return a store in _storemap depending on config |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
698 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
699 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
700 infer the endpoint, based on the remote repository using the same path |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
701 adjustments as git. As an extension, 'http' is supported as well so that |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
702 ``hg serve`` works out of the box. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
703 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
704 https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
705 """ |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
706 lfsurl = repo.ui.config(b'lfs', b'url') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
707 url = util.url(lfsurl or b'') |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
708 if lfsurl is None: |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
709 if remote: |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
710 path = remote |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
711 elif util.safehasattr(repo, b'_subtoppath'): |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
712 # The pull command sets this during the optional update phase, which |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
713 # tells exactly where the pull originated, whether 'paths.default' |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
714 # or explicit. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
715 path = repo._subtoppath |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
716 else: |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
717 # TODO: investigate 'paths.remote:lfsurl' style path customization, |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
718 # and fall back to inferring from 'paths.remote' if unspecified. |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
719 path = repo.ui.config(b'paths', b'default') or b'' |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
720 |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
721 defaulturl = util.url(path) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
722 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
723 # TODO: support local paths as well. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
724 # TODO: consider the ssh -> https transformation that git applies |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
725 if defaulturl.scheme in (b'http', b'https'): |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
726 if defaulturl.path and defaulturl.path[:-1] != b'/': |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
727 defaulturl.path += b'/' |
37691
d241e6632669
lfs: fix the inferred remote store path when using a --prefix
Matt Harbison <matt_harbison@yahoo.com>
parents:
37565
diff
changeset
|
728 defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
729 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
730 url = util.url(bytes(defaulturl)) |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
731 repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
732 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
733 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
734 if scheme not in _storemap: |
41426
02d0a7774882
py3: byteify the LFS blobstore module
Matt Harbison <matt_harbison@yahoo.com>
parents:
40675
diff
changeset
|
735 raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
736 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
737 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
738 |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39457
diff
changeset
|
739 class LfsRemoteError(error.StorageError): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
740 pass |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
741 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41607
diff
changeset
|
742 |
37692
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
743 class LfsCorruptionError(error.Abort): |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
744 """Raised when a corrupt blob is detected, aborting an operation |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
745 |
10e5bb9678f4
lfs: gracefully handle aborts on the server when corrupt blobs are detected
Matt Harbison <matt_harbison@yahoo.com>
parents:
37691
diff
changeset
|
746 It exists to allow specialized handling on the server side.""" |