Mercurial > hg
annotate hgext/largefiles/basestore.py @ 18486:1067a6240f86 stable
largefiles: verify all files in each revision and report errors in any revision
Verify used 'any' and would stop verifying after the first failure in each
changeset.
The exit code only reported the result from the last changeset.
author | Mads Kiilerich <madski@unity3d.com> |
---|---|
date | Mon, 28 Jan 2013 15:19:44 +0100 |
parents | ce5f529deb36 |
children | f1700480bef7 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
9 '''base class for store implementations and store-related utility code''' |
15168 | 10 |
11 import binascii | |
12 import re | |
13 | |
14 from mercurial import util, node, hg | |
15 from mercurial.i18n import _ | |
16 | |
17 import lfutil | |
18 | |
19 class StoreError(Exception): | |
20 '''Raised when there is a problem getting files from or putting | |
21 files to a central store.''' | |
22 def __init__(self, filename, hash, url, detail): | |
23 self.filename = filename | |
24 self.hash = hash | |
25 self.url = url | |
26 self.detail = detail | |
27 | |
28 def longmessage(self): | |
18461
abfbb04fab8e
largefiles: enhance error message to make it more i18n-friendly
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
18155
diff
changeset
|
29 return (_("error getting id %s from url %s for file %s: %s\n") % |
18155
5206af8894a3
largefiles: cleanup of warnings on errors getting largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
17424
diff
changeset
|
30 (self.hash, self.url, self.filename, self.detail)) |
15168 | 31 |
32 def __str__(self): | |
33 return "%s: %s" % (self.url, self.detail) | |
34 | |
35 class basestore(object): | |
36 def __init__(self, ui, repo, url): | |
37 self.ui = ui | |
38 self.repo = repo | |
39 self.url = url | |
40 | |
41 def put(self, source, hash): | |
42 '''Put source file into the store under <filename>/<hash>.''' | |
43 raise NotImplementedError('abstract method') | |
44 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
45 def exists(self, hashes): |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
46 '''Check to see if the store contains the given hashes.''' |
15168 | 47 raise NotImplementedError('abstract method') |
48 | |
49 def get(self, files): | |
50 '''Get the specified largefiles from the store and write to local | |
51 files under repo.root. files is a list of (filename, hash) | |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
52 tuples. Return (success, missing), lists of files successfully |
15168 | 53 downloaded and those not found in the store. success is a list |
54 of (filename, hash) tuples; missing is a list of filenames that | |
55 we could not get. (The detailed error message will already have | |
56 been presented to the user, so missing is just supplied as a | |
57 summary.)''' | |
58 success = [] | |
59 missing = [] | |
60 ui = self.ui | |
61 | |
62 at = 0 | |
63 for filename, hash in files: | |
64 ui.progress(_('getting largefiles'), at, unit='lfile', | |
65 total=len(files)) | |
66 at += 1 | |
67 ui.note(_('getting %s:%s\n') % (filename, hash)) | |
68 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15302
diff
changeset
|
69 storefilename = lfutil.storepath(self.repo, hash) |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
70 tmpfile = util.atomictempfile(storefilename + '.tmp', |
16154
9b072a5f8f92
largefiles: respect store.createmode in basestore.get
Martin Geisler <mg@aragost.com>
parents:
15943
diff
changeset
|
71 createmode=self.repo.store.createmode) |
15168 | 72 |
73 try: | |
74 hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash)) | |
75 except StoreError, err: | |
76 ui.warn(err.longmessage()) | |
77 hhash = "" | |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
78 tmpfile.close() # has probably already been closed! |
15168 | 79 |
80 if hhash != hash: | |
81 if hhash != "": | |
82 ui.warn(_('%s: data corruption (expected %s, got %s)\n') | |
83 % (filename, hash, hhash)) | |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
84 util.unlink(storefilename + '.tmp') |
15168 | 85 missing.append(filename) |
86 continue | |
87 | |
18483
ce5f529deb36
largefiles: don't allow corruption to propagate after detection
Mads Kiilerich <madski@unity3d.com>
parents:
18461
diff
changeset
|
88 util.rename(storefilename + '.tmp', storefilename) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15302
diff
changeset
|
89 lfutil.linktousercache(self.repo, hash) |
15168 | 90 success.append((filename, hhash)) |
91 | |
92 ui.progress(_('getting largefiles'), None) | |
93 return (success, missing) | |
94 | |
95 def verify(self, revs, contents=False): | |
96 '''Verify the existence (and, optionally, contents) of every big | |
97 file revision referenced by every changeset in revs. | |
98 Return 0 if all is well, non-zero on any errors.''' | |
99 write = self.ui.write | |
100 failed = False | |
101 | |
102 write(_('searching %d changesets for largefiles\n') % len(revs)) | |
103 verified = set() # set of (filename, filenode) tuples | |
104 | |
105 for rev in revs: | |
106 cctx = self.repo[rev] | |
107 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | |
108 | |
18486
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
109 for standin in cctx: |
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
110 if self._verifyfile(cctx, cset, contents, standin, verified): |
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
111 failed = True |
15168 | 112 |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
113 numrevs = len(verified) |
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
114 numlfiles = len(set([fname for (fname, fnode) in verified])) |
15168 | 115 if contents: |
116 write(_('verified contents of %d revisions of %d largefiles\n') | |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
117 % (numrevs, numlfiles)) |
15168 | 118 else: |
119 write(_('verified existence of %d revisions of %d largefiles\n') | |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
120 % (numrevs, numlfiles)) |
15168 | 121 |
122 return int(failed) | |
123 | |
124 def _getfile(self, tmpfile, filename, hash): | |
125 '''Fetch one revision of one file from the store and write it | |
126 to tmpfile. Compute the hash of the file on-the-fly as it | |
127 downloads and return the binary hash. Close tmpfile. Raise | |
128 StoreError if unable to download the file (e.g. it does not | |
129 exist in the store).''' | |
130 raise NotImplementedError('abstract method') | |
131 | |
132 def _verifyfile(self, cctx, cset, contents, standin, verified): | |
133 '''Perform the actual verification of a file in the store. | |
134 ''' | |
135 raise NotImplementedError('abstract method') | |
136 | |
137 import localstore, wirestore | |
138 | |
139 _storeprovider = { | |
140 'file': [localstore.localstore], | |
141 'http': [wirestore.wirestore], | |
142 'https': [wirestore.wirestore], | |
143 'ssh': [wirestore.wirestore], | |
144 } | |
145 | |
146 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') | |
147 | |
148 # During clone this function is passed the src's ui object | |
149 # but it needs the dest's ui object so it can read out of | |
150 # the config file. Use repo.ui instead. | |
151 def _openstore(repo, remote=None, put=False): | |
152 ui = repo.ui | |
153 | |
154 if not remote: | |
15943
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
155 lfpullsource = getattr(repo, 'lfpullsource', None) |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
156 if lfpullsource: |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
157 path = ui.expandpath(lfpullsource) |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
158 else: |
f9efb325ea32
largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15319
diff
changeset
|
159 path = ui.expandpath('default-push', 'default') |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
160 |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
161 # ui.expandpath() leaves 'default-push' and 'default' alone if |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
162 # they cannot be expanded: fallback to the empty string, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
163 # meaning the current directory. |
15168 | 164 if path == 'default-push' or path == 'default': |
165 path = '' | |
166 remote = repo | |
167 else: | |
168 remote = hg.peer(repo, {}, path) | |
169 | |
170 # The path could be a scheme so use Mercurial's normal functionality | |
171 # to resolve the scheme to a repository and use its path | |
15169
aa262fff87ac
largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
172 path = util.safehasattr(remote, 'url') and remote.url() or remote.path |
15168 | 173 |
174 match = _scheme_re.match(path) | |
175 if not match: # regular filesystem path | |
176 scheme = 'file' | |
177 else: | |
178 scheme = match.group(1) | |
179 | |
180 try: | |
181 storeproviders = _storeprovider[scheme] | |
182 except KeyError: | |
183 raise util.Abort(_('unsupported URL scheme %r') % scheme) | |
184 | |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
185 for classobj in storeproviders: |
15168 | 186 try: |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
187 return classobj(ui, repo, remote) |
15168 | 188 except lfutil.storeprotonotcapable: |
189 pass | |
190 | |
15302
225d30bacabd
largefiles: string formatting typo in basestore._openstore where comma is used instead of modulo
Hao Lian <hao@fogcreek.com>
parents:
15255
diff
changeset
|
191 raise util.Abort(_('%s does not appear to be a largefile store') % path) |