Mercurial > hg
annotate hgext/lfs/blobstore.py @ 37565:9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
While here, I also checked the lfs.url config directly instead of testing the
scheme, as requested by Yuya.
author | Matt Harbison <matt_harbison@yahoo.com> |
---|---|
date | Wed, 11 Apr 2018 18:23:29 -0400 |
parents | 31a4ea773369 |
children | d241e6632669 |
rev | line source |
---|---|
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
9 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
10 import errno |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
11 import hashlib |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
12 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
14 import re |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
15 import socket |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
16 |
35099
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
17 from mercurial.i18n import _ |
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35098
diff
changeset
|
18 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
19 from mercurial import ( |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
20 error, |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
21 pathutil, |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
22 pycompat, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
23 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
24 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
25 vfs as vfsmod, |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
26 worker, |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
27 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
29 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
30 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
31 # 64 bytes for SHA256 |
36455
9e3cb58c7ab3
py3: make sure regexes are bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35927
diff
changeset
|
32 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
33 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
34 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
35 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
36 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
37 if not _lfsre.match(path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
38 raise error.ProgrammingError('unexpected lfs path: %s' % path) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
39 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
40 |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
41 def walk(self, path=None, onerror=None): |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
42 """Yield (dirpath, [], oids) tuple for blobs under path |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
43 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
44 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
45 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
46 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
47 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
48 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
49 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
50 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
51 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
52 for dirpath, dirs, files in os.walk(self.reljoin(self.base, path or ''), |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
53 onerror=onerror): |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
54 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
55 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
56 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
57 if len(dirpath) == 2: |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
58 oids.extend([dirpath + f for f in files |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
59 if _lfsre.match(dirpath + f)]) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
60 |
35396
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35362
diff
changeset
|
61 yield ('', [], oids) |
35362
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35280
diff
changeset
|
62 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
63 class nullvfs(lfsvfs): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
64 def __init__(self): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
65 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
66 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
67 def exists(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
68 return False |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
69 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
70 def read(self, oid): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
71 # store.read() calls into here if the blob doesn't exist in its |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
72 # self.vfs. Raise the same error as a normal vfs when asked to read a |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
73 # file that doesn't exist. The only difference is the full file path |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
74 # isn't available in the error. |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
75 raise IOError(errno.ENOENT, '%s: No such file or directory' % oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
76 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
77 def walk(self, path=None, onerror=None): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
78 return ('', [], []) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
79 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
80 def write(self, oid, data): |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
81 pass |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
82 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
83 class filewithprogress(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
84 """a file-like object that supports __len__ and read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
85 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
86 Useful to provide progress information for how many bytes are read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
87 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
88 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
89 def __init__(self, fp, callback): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
90 self._fp = fp |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
91 self._callback = callback # func(readsize) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
92 fp.seek(0, os.SEEK_END) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
93 self._len = fp.tell() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
94 fp.seek(0) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
95 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
96 def __len__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
97 return self._len |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
98 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
99 def read(self, size): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
100 if self._fp is None: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
101 return b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
102 data = self._fp.read(size) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
103 if data: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
104 if self._callback: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
105 self._callback(len(data)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
106 else: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
107 self._fp.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
108 self._fp = None |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
109 return data |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
110 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
111 class local(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
114 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
115 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
116 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
117 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
118 def __init__(self, repo): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
119 fullpath = repo.svfs.join('lfs/objects') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
120 self.vfs = lfsvfs(fullpath) |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
121 |
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
122 if repo.ui.configbool('experimental', 'lfs.disableusercache'): |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
123 self.cachevfs = nullvfs() |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
124 else: |
37562
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
125 usercache = lfutil._usercachedir(repo.ui, 'lfs') |
e5cd8d1a094d
lfs: special case the null:// usercache instead of treating it as a url
Matt Harbison <matt_harbison@yahoo.com>
parents:
37518
diff
changeset
|
126 self.cachevfs = lfsvfs(usercache) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
127 self.ui = repo.ui |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
128 |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
129 def open(self, oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
130 """Open a read-only file descriptor to the named blob, in either the |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
131 usercache or the local store.""" |
35537
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
132 # The usercache is the most likely place to hold the file. Commit will |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
133 # write to both it and the local store, as will anything that downloads |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
134 # the blobs. However, things like clone without an update won't |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
135 # populate the local store. For an init + push of a local clone, |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
136 # the usercache is the only place it _could_ be. If not present, the |
58fda95a0202
lfs: add a comment to describe subtle local blobstore open() behavior
Matt Harbison <matt_harbison@yahoo.com>
parents:
35526
diff
changeset
|
137 # missing file msg here will indicate the local repo, not the usercache. |
35525
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
138 if self.cachevfs.exists(oid): |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
139 return self.cachevfs(oid, 'rb') |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
140 |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
141 return self.vfs(oid, 'rb') |
83903433c2eb
lfs: add a local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35478
diff
changeset
|
142 |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
143 def download(self, oid, src): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
144 """Read the blob from the remote source in chunks, verify the content, |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
145 and write to this local blobstore.""" |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
146 sha256 = hashlib.sha256() |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
147 |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
148 with self.vfs(oid, 'wb', atomictemp=True) as fp: |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
149 for chunk in util.filechunkiter(src, size=1048576): |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
150 fp.write(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
151 sha256.update(chunk) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
152 |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
153 realoid = sha256.hexdigest() |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
154 if realoid != oid: |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
155 raise error.Abort(_('corrupt remote lfs object: %s') % oid) |
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
156 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
157 self._linktousercache(oid) |
35551
fa9dd53eb23e
lfs: introduce a localstore method for downloading from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35537
diff
changeset
|
158 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
159 def write(self, oid, data): |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
160 """Write blob to local blobstore. |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
161 |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
162 This should only be called from the filelog during a commit or similar. |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
163 As such, there is no need to verify the data. Imports from a remote |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
164 store must use ``download()`` instead.""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
165 with self.vfs(oid, 'wb', atomictemp=True) as fp: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
166 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
167 |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
168 self._linktousercache(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
169 |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
170 def _linktousercache(self, oid): |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
171 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
172 # the local store on success, but truncate, write and link on failure? |
37517
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
173 if (not self.cachevfs.exists(oid) |
491edf2435a0
lfs: add the ability to disable the usercache
Matt Harbison <matt_harbison@yahoo.com>
parents:
37243
diff
changeset
|
174 and not isinstance(self.cachevfs, nullvfs)): |
35553
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
175 self.ui.note(_('lfs: adding %s to the usercache\n') % oid) |
a77418095530
lfs: remove the verification option when writing to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35552
diff
changeset
|
176 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
177 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
178 def read(self, oid, verify=True): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
179 """Read blob from local blobstore.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
180 if not self.vfs.exists(oid): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
181 blob = self._read(self.cachevfs, oid, verify) |
35477
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
182 |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
183 # Even if revlog will verify the content, it needs to be verified |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
184 # now before making the hardlink to avoid propagating corrupt blobs. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
185 # Don't abort if corruption is detected, because `hg verify` will |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
186 # give more useful info about the corruption- simply don't add the |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
187 # hardlink. |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
188 if verify or hashlib.sha256(blob).hexdigest() == oid: |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
189 self.ui.note(_('lfs: found %s in the usercache\n') % oid) |
bb6a80fc969a
lfs: only hardlink between the usercache and local store if the blob verifies
Matt Harbison <matt_harbison@yahoo.com>
parents:
35476
diff
changeset
|
190 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35473
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
191 else: |
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35440
diff
changeset
|
192 self.ui.note(_('lfs: found %s in the local lfs store\n') % oid) |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
193 blob = self._read(self.vfs, oid, verify) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
194 return blob |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
195 |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
196 def _read(self, vfs, oid, verify): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
197 """Read blob (after verifying) from the given store""" |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
198 blob = vfs.read(oid) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
199 if verify: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
200 _verify(oid, blob) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
201 return blob |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
202 |
37145
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
203 def verify(self, oid): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
204 """Indicate whether or not the hash of the underlying file matches its |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
205 name.""" |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
206 sha256 = hashlib.sha256() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
207 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
208 with self.open(oid) as fp: |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
209 for chunk in util.filechunkiter(fp, size=1048576): |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
210 sha256.update(chunk) |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
211 |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
212 return oid == sha256.hexdigest() |
56c7cd067477
lfs: add a blob verification method to the local store
Matt Harbison <matt_harbison@yahoo.com>
parents:
36926
diff
changeset
|
213 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
214 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
215 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
216 False otherwise.""" |
35280
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35100
diff
changeset
|
217 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
218 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
219 class _gitlfsremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
220 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
221 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
222 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
223 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
224 baseurl, authinfo = url.authinfo() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
225 self.baseurl = baseurl.rstrip('/') |
35440
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
226 useragent = repo.ui.config('experimental', 'lfs.user-agent') |
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35439
diff
changeset
|
227 if not useragent: |
35733
3d48ae1aaa5e
lfs: default the User-Agent header for blob transfers to 'git-lfs'
Matt Harbison <matt_harbison@yahoo.com>
parents:
35732
diff
changeset
|
228 useragent = 'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
35439
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35433
diff
changeset
|
229 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
35100
07e97998d385
lfs: register config options
Matt Harbison <matt_harbison@yahoo.com>
parents:
35099
diff
changeset
|
230 self.retry = ui.configint('lfs', 'retry') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
231 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
232 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
233 """Batch upload from local to remote blobstore.""" |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
234 self._batch(_deduplicate(pointers), fromstore, 'upload') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
235 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
236 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
237 """Batch download from remote to local blostore.""" |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
238 self._batch(_deduplicate(pointers), tostore, 'download') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
239 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
240 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
241 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
242 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
243 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
244 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
245 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
246 objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
247 requestdata = json.dumps({ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
248 'objects': objects, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
249 'operation': action, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
250 }) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
251 batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
252 data=requestdata) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
253 batchreq.add_header('Accept', 'application/vnd.git-lfs+json') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
254 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
255 try: |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
256 rsp = self.urlopener.open(batchreq) |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
257 rawjson = rsp.read() |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
258 except util.urlerr.httperror as ex: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
259 raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
260 % (ex, action)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
261 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
262 response = json.loads(rawjson) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
263 except ValueError: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
264 raise LfsRemoteError(_('LFS server returns invalid JSON: %s') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
265 % rawjson) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
266 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
267 if self.ui.debugflag: |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
268 self.ui.debug('Status: %d\n' % rsp.status) |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
269 # lfs-test-server and hg serve return headers in different order |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
270 self.ui.debug('%s\n' |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
271 % '\n'.join(sorted(str(rsp.info()).splitlines()))) |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
272 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
273 if 'objects' in response: |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
274 response['objects'] = sorted(response['objects'], |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
275 key=lambda p: p['oid']) |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
276 self.ui.debug('%s\n' |
37146
c37c47e47a95
test-lfs: drop trailing ', ' item separators from debug JSON output
Matt Harbison <matt_harbison@yahoo.com>
parents:
37145
diff
changeset
|
277 % json.dumps(response, indent=2, |
c37c47e47a95
test-lfs: drop trailing ', ' item separators from debug JSON output
Matt Harbison <matt_harbison@yahoo.com>
parents:
37145
diff
changeset
|
278 separators=('', ': '), sort_keys=True)) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
279 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
280 return response |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
281 |
35666 | 282 def _checkforservererror(self, pointers, responses, action): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
283 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
284 |
35694
8a23082f4d93
lfs: correct documentation typo
Matt Harbison <matt_harbison@yahoo.com>
parents:
35666
diff
changeset
|
285 Raises LfsRemoteError if any objects have an error""" |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
286 for response in responses: |
35666 | 287 # The server should return 404 when objects cannot be found. Some |
288 # server implementation (ex. lfs-test-server) does not set "error" | |
289 # but just removes "download" from "actions". Treat that case | |
290 # as the same as 404 error. | |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
291 if 'error' not in response: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
292 if (action == 'download' |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
293 and action not in response.get('actions', [])): |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
294 code = 404 |
35695
dd672e3d059f
lfs: raise an error if the server sends an unsolicited oid
Matt Harbison <matt_harbison@yahoo.com>
parents:
35694
diff
changeset
|
295 else: |
37242
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
296 continue |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
297 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
298 # An error dict without a code doesn't make much sense, so |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
299 # treat as a server error. |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
300 code = response.get('error').get('code', 500) |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
301 |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
302 ptrmap = {p.oid(): p for p in pointers} |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
303 p = ptrmap.get(response['oid'], None) |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
304 if p: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
305 filename = getattr(p, 'filename', 'unknown') |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
306 errors = { |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
307 404: 'The object does not exist', |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
308 410: 'The object was removed by the owner', |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
309 422: 'Validation error', |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
310 500: 'Internal server error', |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
311 } |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
312 msg = errors.get(code, 'status code %d' % code) |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
313 raise LfsRemoteError(_('LFS server error for "%s": %s') |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
314 % (filename, msg)) |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
315 else: |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
316 raise LfsRemoteError( |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
317 _('LFS server error. Unsolicited response for oid %s') |
67db84842356
lfs: improve the client message when the server signals an object error
Matt Harbison <matt_harbison@yahoo.com>
parents:
37217
diff
changeset
|
318 % response['oid']) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
319 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
320 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
321 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
322 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
323 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
324 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
325 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
326 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
327 # Scan errors from objects - fail early |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
328 objects = response.get('objects', []) |
35666 | 329 self._checkforservererror(pointers, objects, action) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
330 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
331 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
332 # objects which exist in the server. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
333 filteredobjects = [o for o in objects if action in o.get('actions', [])] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
334 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
335 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
336 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
337 def _basictransfer(self, obj, action, localstore): |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
338 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
339 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
340 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
341 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
342 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
343 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
344 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
345 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
346 """ |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
347 oid = pycompat.bytestr(obj['oid']) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
348 |
36601
4da09b46451e
lfs: add some bytestring wrappers in blobstore.py
Augie Fackler <augie@google.com>
parents:
36455
diff
changeset
|
349 href = pycompat.bytestr(obj['actions'][action].get('href')) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
350 headers = obj['actions'][action].get('header', {}).items() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
351 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
352 request = util.urlreq.request(href) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
353 if action == 'upload': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
354 # If uploading blobs, read data from local blobstore. |
37217
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
355 if not localstore.verify(oid): |
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
356 raise error.Abort(_('detected corrupt lfs object: %s') % oid, |
b00bd974eef5
lfs: drop a duplicate blob verification method
Matt Harbison <matt_harbison@yahoo.com>
parents:
37146
diff
changeset
|
357 hint=_('run hg verify')) |
35526
e8f80529abeb
lfs: use the local store method for opening a blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35525
diff
changeset
|
358 request.data = filewithprogress(localstore.open(oid), None) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
359 request.get_method = lambda: 'PUT' |
37243
3e293808e835
lfs: add the 'Content-Type' header called out in the file transfer spec
Matt Harbison <matt_harbison@yahoo.com>
parents:
37242
diff
changeset
|
360 request.add_header('Content-Type', 'application/octet-stream') |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
361 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
362 for k, v in headers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
363 request.add_header(k, v) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
364 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
365 response = b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
366 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
367 req = self.urlopener.open(request) |
36926
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
368 |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
369 if self.ui.debugflag: |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
370 self.ui.debug('Status: %d\n' % req.status) |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
371 # lfs-test-server and hg serve return headers in different order |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
372 self.ui.debug('%s\n' |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
373 % '\n'.join(sorted(str(req.info()).splitlines()))) |
0dcf50dc90b6
lfs: debug print HTTP headers and JSON payload received from the server
Matt Harbison <matt_harbison@yahoo.com>
parents:
36601
diff
changeset
|
374 |
35552
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
375 if action == 'download': |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
376 # If downloading blobs, store downloaded data to local blobstore |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
377 localstore.download(oid, req) |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
378 else: |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
379 while True: |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
380 data = req.read(1048576) |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
381 if not data: |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
382 break |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
383 response += data |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
384 if response: |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
385 self.ui.debug('lfs %s response: %s' % (action, response)) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
386 except util.urlerr.httperror as ex: |
35734
b4e1d0654736
lfs: dump the full response on httperror in debug mode
Matt Harbison <matt_harbison@yahoo.com>
parents:
35733
diff
changeset
|
387 if self.ui.debugflag: |
35753
069df0b952e8
lfs: separate a debug message from the subsequent abort message
Matt Harbison <matt_harbison@yahoo.com>
parents:
35734
diff
changeset
|
388 self.ui.debug('%s: %s\n' % (oid, ex.read())) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
389 raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
390 % (ex, oid, action)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
391 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
392 def _batch(self, pointers, localstore, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
393 if action not in ['upload', 'download']: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
394 raise error.ProgrammingError('invalid Git-LFS action: %s' % action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
395 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
396 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
397 objects = self._extractobjects(response, pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
398 total = sum(x.get('size', 0) for x in objects) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
399 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
400 for obj in objects: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
401 sizes[obj.get('oid')] = obj.get('size', 0) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
402 topic = {'upload': _('lfs uploading'), |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
403 'download': _('lfs downloading')}[action] |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
404 if len(objects) > 1: |
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
405 self.ui.note(_('lfs: need to transfer %d objects (%s)\n') |
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
406 % (len(objects), util.bytecount(total))) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
407 self.ui.progress(topic, 0, total=total) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
408 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
409 for obj in chunk: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
410 objsize = obj.get('size', 0) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
411 if self.ui.verbose: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
412 if action == 'download': |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
413 msg = _('lfs: downloading %s (%s)\n') |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
414 elif action == 'upload': |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
415 msg = _('lfs: uploading %s (%s)\n') |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
416 self.ui.note(msg % (obj.get('oid'), |
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
417 util.bytecount(objsize))) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
418 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
419 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
420 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
421 self._basictransfer(obj, action, localstore) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
422 yield 1, obj.get('oid') |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
423 break |
35475
b0c01a5ee35c
lfs: narrow the exceptions that trigger a transfer retry
Matt Harbison <matt_harbison@yahoo.com>
parents:
35473
diff
changeset
|
424 except socket.error as ex: |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
425 if retry > 0: |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
426 self.ui.note( |
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
427 _('lfs: failed: %r (remaining retry %d)\n') |
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
428 % (ex, retry)) |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
429 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
430 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
431 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
432 |
35732
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
433 # Until https multiplexing gets sorted out |
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
434 if self.ui.configbool('experimental', 'lfs.worker-enable'): |
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
435 oids = worker.worker(self.ui, 0.1, transfer, (), |
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
436 sorted(objects, key=lambda o: o.get('oid'))) |
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
437 else: |
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
438 oids = transfer(sorted(objects, key=lambda o: o.get('oid'))) |
10e62d5efa73
lfs: default to not using workers for upload/download
Matt Harbison <matt_harbison@yahoo.com>
parents:
35695
diff
changeset
|
439 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
440 processed = 0 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
441 blobs = 0 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
442 for _one, oid in oids: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
443 processed += sizes[oid] |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
444 blobs += 1 |
35433
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35396
diff
changeset
|
445 self.ui.progress(topic, processed, total=total) |
35478
5a73a0446afd
lfs: use ui.note() and ui.debug() instead of ui.write() and their flags
Matt Harbison <matt_harbison@yahoo.com>
parents:
35477
diff
changeset
|
446 self.ui.note(_('lfs: processed: %s\n') % oid) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
447 self.ui.progress(topic, pos=None, total=total) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
448 |
35881
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
449 if blobs > 0: |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
450 if action == 'upload': |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
451 self.ui.status(_('lfs: uploaded %d files (%s)\n') |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
452 % (blobs, util.bytecount(processed))) |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
453 # TODO: coalesce the download requests, and comment this in |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
454 #elif action == 'download': |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
455 # self.ui.status(_('lfs: downloaded %d files (%s)\n') |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
456 # % (blobs, util.bytecount(processed))) |
fa993c3c8462
lfs: emit a status message to indicate how many blobs were uploaded
Matt Harbison <matt_harbison@yahoo.com>
parents:
35753
diff
changeset
|
457 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
458 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
459 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
460 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
461 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
462 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
463 h.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
464 getattr(h, "close_all", lambda : None)() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
465 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
466 class _dummyremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
467 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
468 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
469 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
470 fullpath = repo.vfs.join('lfs', url.path) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
471 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
472 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
473 def writebatch(self, pointers, fromstore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
474 for p in _deduplicate(pointers): |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
475 content = fromstore.read(p.oid(), verify=True) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
476 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
477 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
478 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
479 def readbatch(self, pointers, tostore): |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
480 for p in _deduplicate(pointers): |
35552
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
481 with self.vfs(p.oid(), 'rb') as fp: |
fd610befc37f
lfs: use the localstore download method to transfer from remote stores
Matt Harbison <matt_harbison@yahoo.com>
parents:
35551
diff
changeset
|
482 tostore.download(p.oid(), fp) |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
483 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
484 class _nullremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
485 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
486 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
487 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
488 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
489 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
490 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
491 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
492 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
493 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
494 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
495 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
496 class _promptremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
497 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
498 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
499 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
500 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
501 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
502 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
503 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
504 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
505 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
506 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
507 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
508 def _prompt(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
509 raise error.Abort(_('lfs.url needs to be configured')) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
510 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
511 _storemap = { |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
512 'https': _gitlfsremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
513 'http': _gitlfsremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
514 'file': _dummyremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
515 'null': _nullremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
516 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
517 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
518 |
35927
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
519 def _deduplicate(pointers): |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
520 """Remove any duplicate oids that exist in the list""" |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
521 reduced = util.sortdict() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
522 for p in pointers: |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
523 reduced[p.oid()] = p |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
524 return reduced.values() |
9b413478f261
lfs: deduplicate oids in the transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35881
diff
changeset
|
525 |
35476
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
526 def _verify(oid, content): |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
527 realoid = hashlib.sha256(content).hexdigest() |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
528 if realoid != oid: |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
529 raise error.Abort(_('detected corrupt lfs object: %s') % oid, |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
530 hint=_('run hg verify')) |
417e8e040102
lfs: verify lfs object content when transferring to and from the remote store
Matt Harbison <matt_harbison@yahoo.com>
parents:
35475
diff
changeset
|
531 |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
532 def remote(repo, remote=None): |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
533 """remotestore factory. return a store in _storemap depending on config |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
534 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
535 If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
536 infer the endpoint, based on the remote repository using the same path |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
537 adjustments as git. As an extension, 'http' is supported as well so that |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
538 ``hg serve`` works out of the box. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
539 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
540 https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
541 """ |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
542 lfsurl = repo.ui.config('lfs', 'url') |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
543 url = util.url(lfsurl or '') |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
544 if lfsurl is None: |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
545 if remote: |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
546 path = remote |
37564
31a4ea773369
lfs: infer the blob store URL from an explicit push dest or default-push
Matt Harbison <matt_harbison@yahoo.com>
parents:
37563
diff
changeset
|
547 elif util.safehasattr(repo, '_subtoppath'): |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
548 # The pull command sets this during the optional update phase, which |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
549 # tells exactly where the pull originated, whether 'paths.default' |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
550 # or explicit. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
551 path = repo._subtoppath |
37563
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
552 else: |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
553 # TODO: investigate 'paths.remote:lfsurl' style path customization, |
be1cc65bdb1c
lfs: infer the blob store URL from an explicit pull source
Matt Harbison <matt_harbison@yahoo.com>
parents:
37562
diff
changeset
|
554 # and fall back to inferring from 'paths.remote' if unspecified. |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
555 path = repo.ui.config('paths', 'default') or '' |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
556 |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
557 defaulturl = util.url(path) |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
558 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
559 # TODO: support local paths as well. |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
560 # TODO: consider the ssh -> https transformation that git applies |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
561 if defaulturl.scheme in (b'http', b'https'): |
37565
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
562 if defaulturl.path and defaulturl.path[:-1] != b'/': |
9c7a25ef5b49
lfs: handle paths that don't end with '/' when inferring the blob store
Matt Harbison <matt_harbison@yahoo.com>
parents:
37564
diff
changeset
|
563 defaulturl.path += b'/' |
37518
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
564 defaulturl.path = defaulturl.path or b'' + b'.git/info/lfs' |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
565 |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
566 url = util.url(bytes(defaulturl)) |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
567 repo.ui.note(_('lfs: assuming remote store: %s\n') % url) |
092eff6833a7
lfs: infer the blob store URL from paths.default
Matt Harbison <matt_harbison@yahoo.com>
parents:
37517
diff
changeset
|
568 |
35098
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
569 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
570 if scheme not in _storemap: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
571 raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
572 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
573 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
574 class LfsRemoteError(error.RevlogError): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
575 pass |