Mercurial > hg
annotate hgext/largefiles/basestore.py @ 24118:76f6ae06ddf5
revlog: add "iscensored()" to revlog public API
The iscensored method will be used by the exchange layer to reject
nonconforming deltas involving censored revisions (and to produce
conforming deltas).
For background and broader design of the censorship feature, see:
http://mercurial.selenic.com/wiki/CensorPlan
author | Mike Edgar <adgar@google.com> |
---|---|
date | Fri, 23 Jan 2015 17:01:39 -0500 |
parents | 164bd5218ddb |
children | 328739ea70c3 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
9 '''base class for store implementations and store-related utility code''' |
15168 | 10 |
11 import re | |
12 | |
13 from mercurial import util, node, hg | |
14 from mercurial.i18n import _ | |
15 | |
16 import lfutil | |
17 | |
18 class StoreError(Exception): | |
19 '''Raised when there is a problem getting files from or putting | |
20 files to a central store.''' | |
21 def __init__(self, filename, hash, url, detail): | |
22 self.filename = filename | |
23 self.hash = hash | |
24 self.url = url | |
25 self.detail = detail | |
26 | |
27 def longmessage(self): | |
18461
abfbb04fab8e
largefiles: enhance error message to make it more i18n-friendly
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
18155
diff
changeset
|
28 return (_("error getting id %s from url %s for file %s: %s\n") % |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
29 (self.hash, util.hidepassword(self.url), self.filename, |
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
30 self.detail)) |
15168 | 31 |
32 def __str__(self): | |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
33 return "%s: %s" % (util.hidepassword(self.url), self.detail) |
15168 | 34 |
35 class basestore(object): | |
36 def __init__(self, ui, repo, url): | |
37 self.ui = ui | |
38 self.repo = repo | |
39 self.url = url | |
40 | |
41 def put(self, source, hash): | |
19007
266b5fb72f26
largefiles: 'put' should store 'source' file in under 'hash', also in localstore
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
42 '''Put source file into the store so it can be retrieved by hash.''' |
15168 | 43 raise NotImplementedError('abstract method') |
44 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
45 def exists(self, hashes): |
18573
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18546
diff
changeset
|
46 '''Check to see if the store contains the given hashes. Given an |
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18546
diff
changeset
|
47 iterable of hashes it returns a mapping from hash to bool.''' |
15168 | 48 raise NotImplementedError('abstract method') |
49 | |
50 def get(self, files): | |
51 '''Get the specified largefiles from the store and write to local | |
52 files under repo.root. files is a list of (filename, hash) | |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
53 tuples. Return (success, missing), lists of files successfully |
15168 | 54 downloaded and those not found in the store. success is a list |
55 of (filename, hash) tuples; missing is a list of filenames that | |
56 we could not get. (The detailed error message will already have | |
57 been presented to the user, so missing is just supplied as a | |
58 summary.)''' | |
59 success = [] | |
60 missing = [] | |
61 ui = self.ui | |
62 | |
63 at = 0 | |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
64 available = self.exists(set(hash for (_filename, hash) in files)) |
15168 | 65 for filename, hash in files: |
66 ui.progress(_('getting largefiles'), at, unit='lfile', | |
67 total=len(files)) | |
68 at += 1 | |
69 ui.note(_('getting %s:%s\n') % (filename, hash)) | |
70 | |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
71 if not available.get(hash): |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
72 ui.warn(_('%s: largefile %s not available from %s\n') |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
73 % (filename, hash, util.hidepassword(self.url))) |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
74 missing.append(filename) |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
75 continue |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
76 |
19918
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
77 if self._gethash(filename, hash): |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
78 success.append((filename, hash)) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
79 else: |
15168 | 80 missing.append(filename) |
81 | |
82 ui.progress(_('getting largefiles'), None) | |
83 return (success, missing) | |
84 | |
19918
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
85 def _gethash(self, filename, hash): |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
86 """Get file with the provided hash and store it in the local repo's |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
87 store and in the usercache. |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
88 filename is for informational messages only. |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
89 """ |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
90 util.makedirs(lfutil.storepath(self.repo, '')) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
91 storefilename = lfutil.storepath(self.repo, hash) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
92 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
93 tmpname = storefilename + '.tmp' |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
94 tmpfile = util.atomictempfile(tmpname, |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
95 createmode=self.repo.store.createmode) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
96 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
97 try: |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
98 gothash = self._getfile(tmpfile, filename, hash) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
99 except StoreError, err: |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
100 self.ui.warn(err.longmessage()) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
101 gothash = "" |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
102 tmpfile.close() |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
103 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
104 if gothash != hash: |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
105 if gothash != "": |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
106 self.ui.warn(_('%s: data corruption (expected %s, got %s)\n') |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
107 % (filename, hash, gothash)) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
108 util.unlink(tmpname) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
109 return False |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
110 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
111 util.rename(tmpname, storefilename) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
112 lfutil.linktousercache(self.repo, hash) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
113 return True |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
114 |
15168 | 115 def verify(self, revs, contents=False): |
116 '''Verify the existence (and, optionally, contents) of every big | |
117 file revision referenced by every changeset in revs. | |
118 Return 0 if all is well, non-zero on any errors.''' | |
119 failed = False | |
120 | |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
121 self.ui.status(_('searching %d changesets for largefiles\n') % |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
122 len(revs)) |
15168 | 123 verified = set() # set of (filename, filenode) tuples |
124 | |
125 for rev in revs: | |
126 cctx = self.repo[rev] | |
127 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | |
128 | |
18486
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
129 for standin in cctx: |
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
130 if self._verifyfile(cctx, cset, contents, standin, verified): |
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
131 failed = True |
15168 | 132 |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
133 numrevs = len(verified) |
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
134 numlfiles = len(set([fname for (fname, fnode) in verified])) |
15168 | 135 if contents: |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
136 self.ui.status( |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
137 _('verified contents of %d revisions of %d largefiles\n') |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
138 % (numrevs, numlfiles)) |
15168 | 139 else: |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
140 self.ui.status( |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
141 _('verified existence of %d revisions of %d largefiles\n') |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
142 % (numrevs, numlfiles)) |
15168 | 143 return int(failed) |
144 | |
145 def _getfile(self, tmpfile, filename, hash): | |
146 '''Fetch one revision of one file from the store and write it | |
147 to tmpfile. Compute the hash of the file on-the-fly as it | |
18999
c1b5f9c4d989
largefiles: refactoring - return hex from _getfile and copyandhash
Mads Kiilerich <madski@unity3d.com>
parents:
18731
diff
changeset
|
148 downloads and return the hash. Close tmpfile. Raise |
15168 | 149 StoreError if unable to download the file (e.g. it does not |
150 exist in the store).''' | |
151 raise NotImplementedError('abstract method') | |
152 | |
153 def _verifyfile(self, cctx, cset, contents, standin, verified): | |
154 '''Perform the actual verification of a file in the store. | |
18574
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
155 'cset' is only used in warnings. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
156 'contents' controls verification of content hash. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
157 'standin' is the standin path of the largefile to verify. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
158 'verified' is maintained as a set of already verified files. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
159 Returns _true_ if it is a standin and any problems are found! |
15168 | 160 ''' |
161 raise NotImplementedError('abstract method') | |
162 | |
163 import localstore, wirestore | |
164 | |
165 _storeprovider = { | |
166 'file': [localstore.localstore], | |
167 'http': [wirestore.wirestore], | |
168 'https': [wirestore.wirestore], | |
169 'ssh': [wirestore.wirestore], | |
170 } | |
171 | |
172 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') | |
173 | |
174 # During clone this function is passed the src's ui object | |
175 # but it needs the dest's ui object so it can read out of | |
176 # the config file. Use repo.ui instead. | |
177 def _openstore(repo, remote=None, put=False): | |
178 ui = repo.ui | |
179 | |
180 if not remote: | |
15943
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
181 lfpullsource = getattr(repo, 'lfpullsource', None) |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
182 if lfpullsource: |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
183 path = ui.expandpath(lfpullsource) |
23941
164bd5218ddb
largefiles: use 'default' path for pulling largefiles, not 'default-push'
Mads Kiilerich <madski@unity3d.com>
parents:
19950
diff
changeset
|
184 elif put: |
164bd5218ddb
largefiles: use 'default' path for pulling largefiles, not 'default-push'
Mads Kiilerich <madski@unity3d.com>
parents:
19950
diff
changeset
|
185 path = ui.expandpath('default-push', 'default') |
15943
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
186 else: |
23941
164bd5218ddb
largefiles: use 'default' path for pulling largefiles, not 'default-push'
Mads Kiilerich <madski@unity3d.com>
parents:
19950
diff
changeset
|
187 path = ui.expandpath('default') |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
188 |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
189 # ui.expandpath() leaves 'default-push' and 'default' alone if |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
190 # they cannot be expanded: fallback to the empty string, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
191 # meaning the current directory. |
15168 | 192 if path == 'default-push' or path == 'default': |
193 path = '' | |
194 remote = repo | |
195 else: | |
18489
f1700480bef7
largefiles: allow use of urls with #revision
Mads Kiilerich <madski@unity3d.com>
parents:
18486
diff
changeset
|
196 path, _branches = hg.parseurl(path) |
15168 | 197 remote = hg.peer(repo, {}, path) |
198 | |
199 # The path could be a scheme so use Mercurial's normal functionality | |
200 # to resolve the scheme to a repository and use its path | |
15169
aa262fff87ac
largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
201 path = util.safehasattr(remote, 'url') and remote.url() or remote.path |
15168 | 202 |
203 match = _scheme_re.match(path) | |
204 if not match: # regular filesystem path | |
205 scheme = 'file' | |
206 else: | |
207 scheme = match.group(1) | |
208 | |
209 try: | |
210 storeproviders = _storeprovider[scheme] | |
211 except KeyError: | |
212 raise util.Abort(_('unsupported URL scheme %r') % scheme) | |
213 | |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
214 for classobj in storeproviders: |
15168 | 215 try: |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
216 return classobj(ui, repo, remote) |
15168 | 217 except lfutil.storeprotonotcapable: |
218 pass | |
219 | |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
220 raise util.Abort(_('%s does not appear to be a largefile store') % |
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
221 util.hidepassword(path)) |