Mercurial > hg
annotate hgext/largefiles/basestore.py @ 19008:9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
This avoids a lot of expensive roundtrips to remote repositories ... but might
be slightly slower for local operations.
This will also change some aborts on missing files to warnings. That will in
some situations make it possible to continue working on a repository with
missing largefiles.
author | Mads Kiilerich <madski@unity3d.com> |
---|---|
date | Mon, 15 Apr 2013 23:37:43 +0200 |
parents | 266b5fb72f26 |
children | ae65192fd6b4 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
9 '''base class for store implementations and store-related utility code''' |
15168 | 10 |
11 import re | |
12 | |
13 from mercurial import util, node, hg | |
14 from mercurial.i18n import _ | |
15 | |
16 import lfutil | |
17 | |
18 class StoreError(Exception): | |
19 '''Raised when there is a problem getting files from or putting | |
20 files to a central store.''' | |
21 def __init__(self, filename, hash, url, detail): | |
22 self.filename = filename | |
23 self.hash = hash | |
24 self.url = url | |
25 self.detail = detail | |
26 | |
27 def longmessage(self): | |
18461
abfbb04fab8e
largefiles: enhance error message to make it more i18n-friendly
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
18155
diff
changeset
|
28 return (_("error getting id %s from url %s for file %s: %s\n") % |
18155
5206af8894a3
largefiles: cleanup of warnings on errors getting largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
17424
diff
changeset
|
29 (self.hash, self.url, self.filename, self.detail)) |
15168 | 30 |
31 def __str__(self): | |
32 return "%s: %s" % (self.url, self.detail) | |
33 | |
34 class basestore(object): | |
35 def __init__(self, ui, repo, url): | |
36 self.ui = ui | |
37 self.repo = repo | |
38 self.url = url | |
39 | |
40 def put(self, source, hash): | |
19007
266b5fb72f26
largefiles: 'put' should store 'source' file in under 'hash', also in localstore
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
41 '''Put source file into the store so it can be retrieved by hash.''' |
15168 | 42 raise NotImplementedError('abstract method') |
43 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
44 def exists(self, hashes): |
18573
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18546
diff
changeset
|
45 '''Check to see if the store contains the given hashes. Given an |
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18546
diff
changeset
|
46 iterable of hashes it returns a mapping from hash to bool.''' |
15168 | 47 raise NotImplementedError('abstract method') |
48 | |
49 def get(self, files): | |
50 '''Get the specified largefiles from the store and write to local | |
51 files under repo.root. files is a list of (filename, hash) | |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
52 tuples. Return (success, missing), lists of files successfully |
15168 | 53 downloaded and those not found in the store. success is a list |
54 of (filename, hash) tuples; missing is a list of filenames that | |
55 we could not get. (The detailed error message will already have | |
56 been presented to the user, so missing is just supplied as a | |
57 summary.)''' | |
58 success = [] | |
59 missing = [] | |
60 ui = self.ui | |
61 | |
18725
0ac00315875f
largefiles: don't assume that .hg/largefiles/ still exists
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
62 util.makedirs(lfutil.storepath(self.repo, '')) |
0ac00315875f
largefiles: don't assume that .hg/largefiles/ still exists
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
63 |
15168 | 64 at = 0 |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
65 available = self.exists(set(hash for (_filename, hash) in files)) |
15168 | 66 for filename, hash in files: |
67 ui.progress(_('getting largefiles'), at, unit='lfile', | |
68 total=len(files)) | |
69 at += 1 | |
70 ui.note(_('getting %s:%s\n') % (filename, hash)) | |
71 | |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
72 if not available.get(hash): |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
73 ui.warn(_('%s: largefile %s not available from %s\n') |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
74 % (filename, hash, self.url)) |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
75 missing.append(filename) |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
76 continue |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
77 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15302
diff
changeset
|
78 storefilename = lfutil.storepath(self.repo, hash) |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
79 tmpfile = util.atomictempfile(storefilename + '.tmp', |
16154
9b072a5f8f92
largefiles: respect store.createmode in basestore.get
Martin Geisler <mg@aragost.com>
parents:
15943
diff
changeset
|
80 createmode=self.repo.store.createmode) |
15168 | 81 |
82 try: | |
18999
c1b5f9c4d989
largefiles: refactoring - return hex from _getfile and copyandhash
Mads Kiilerich <madski@unity3d.com>
parents:
18731
diff
changeset
|
83 hhash = self._getfile(tmpfile, filename, hash) |
15168 | 84 except StoreError, err: |
85 ui.warn(err.longmessage()) | |
86 hhash = "" | |
19003
ad993cb7bbb1
largefiles: don't close the fd passed to store._getfile
Mads Kiilerich <madski@unity3d.com>
parents:
18999
diff
changeset
|
87 tmpfile.close() |
15168 | 88 |
89 if hhash != hash: | |
90 if hhash != "": | |
91 ui.warn(_('%s: data corruption (expected %s, got %s)\n') | |
92 % (filename, hash, hhash)) | |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
93 util.unlink(storefilename + '.tmp') |
15168 | 94 missing.append(filename) |
95 continue | |
96 | |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
97 util.rename(storefilename + '.tmp', storefilename) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15302
diff
changeset
|
98 lfutil.linktousercache(self.repo, hash) |
15168 | 99 success.append((filename, hhash)) |
100 | |
101 ui.progress(_('getting largefiles'), None) | |
102 return (success, missing) | |
103 | |
104 def verify(self, revs, contents=False): | |
105 '''Verify the existence (and, optionally, contents) of every big | |
106 file revision referenced by every changeset in revs. | |
107 Return 0 if all is well, non-zero on any errors.''' | |
108 failed = False | |
109 | |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
110 self.ui.status(_('searching %d changesets for largefiles\n') % |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
111 len(revs)) |
15168 | 112 verified = set() # set of (filename, filenode) tuples |
113 | |
114 for rev in revs: | |
115 cctx = self.repo[rev] | |
116 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | |
117 | |
18486
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
118 for standin in cctx: |
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
119 if self._verifyfile(cctx, cset, contents, standin, verified): |
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
120 failed = True |
15168 | 121 |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
122 numrevs = len(verified) |
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
123 numlfiles = len(set([fname for (fname, fnode) in verified])) |
15168 | 124 if contents: |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
125 self.ui.status( |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
126 _('verified contents of %d revisions of %d largefiles\n') |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
127 % (numrevs, numlfiles)) |
15168 | 128 else: |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
129 self.ui.status( |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
130 _('verified existence of %d revisions of %d largefiles\n') |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
131 % (numrevs, numlfiles)) |
15168 | 132 return int(failed) |
133 | |
134 def _getfile(self, tmpfile, filename, hash): | |
135 '''Fetch one revision of one file from the store and write it | |
136 to tmpfile. Compute the hash of the file on-the-fly as it | |
18999
c1b5f9c4d989
largefiles: refactoring - return hex from _getfile and copyandhash
Mads Kiilerich <madski@unity3d.com>
parents:
18731
diff
changeset
|
137 downloads and return the hash. Close tmpfile. Raise |
15168 | 138 StoreError if unable to download the file (e.g. it does not |
139 exist in the store).''' | |
140 raise NotImplementedError('abstract method') | |
141 | |
142 def _verifyfile(self, cctx, cset, contents, standin, verified): | |
143 '''Perform the actual verification of a file in the store. | |
18574
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
144 'cset' is only used in warnings. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
145 'contents' controls verification of content hash. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
146 'standin' is the standin path of the largefile to verify. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
147 'verified' is maintained as a set of already verified files. |
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
148 Returns _true_ if it is a standin and any problems are found! |
15168 | 149 ''' |
150 raise NotImplementedError('abstract method') | |
151 | |
152 import localstore, wirestore | |
153 | |
154 _storeprovider = { | |
155 'file': [localstore.localstore], | |
156 'http': [wirestore.wirestore], | |
157 'https': [wirestore.wirestore], | |
158 'ssh': [wirestore.wirestore], | |
159 } | |
160 | |
161 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') | |
162 | |
163 # During clone this function is passed the src's ui object | |
164 # but it needs the dest's ui object so it can read out of | |
165 # the config file. Use repo.ui instead. | |
166 def _openstore(repo, remote=None, put=False): | |
167 ui = repo.ui | |
168 | |
169 if not remote: | |
15943
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
170 lfpullsource = getattr(repo, 'lfpullsource', None) |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
171 if lfpullsource: |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
172 path = ui.expandpath(lfpullsource) |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
173 else: |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
174 path = ui.expandpath('default-push', 'default') |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
175 |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
176 # ui.expandpath() leaves 'default-push' and 'default' alone if |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
177 # they cannot be expanded: fallback to the empty string, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
178 # meaning the current directory. |
15168 | 179 if path == 'default-push' or path == 'default': |
180 path = '' | |
181 remote = repo | |
182 else: | |
18489
f1700480bef7
largefiles: allow use of urls with #revision
Mads Kiilerich <madski@unity3d.com>
parents:
18486
diff
changeset
|
183 path, _branches = hg.parseurl(path) |
15168 | 184 remote = hg.peer(repo, {}, path) |
185 | |
186 # The path could be a scheme so use Mercurial's normal functionality | |
187 # to resolve the scheme to a repository and use its path | |
15169
aa262fff87ac
largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
188 path = util.safehasattr(remote, 'url') and remote.url() or remote.path |
15168 | 189 |
190 match = _scheme_re.match(path) | |
191 if not match: # regular filesystem path | |
192 scheme = 'file' | |
193 else: | |
194 scheme = match.group(1) | |
195 | |
196 try: | |
197 storeproviders = _storeprovider[scheme] | |
198 except KeyError: | |
199 raise util.Abort(_('unsupported URL scheme %r') % scheme) | |
200 | |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
201 for classobj in storeproviders: |
15168 | 202 try: |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
203 return classobj(ui, repo, remote) |
15168 | 204 except lfutil.storeprotonotcapable: |
205 pass | |
206 | |
15302
225d30bacabd
largefiles: string formatting typo in basestore._openstore where comma is used instead of modulo
Hao Lian <hao@fogcreek.com>
parents:
15255
diff
changeset
|
207 raise util.Abort(_('%s does not appear to be a largefile store') % path) |