Mercurial > hg
annotate hgext/largefiles/remotestore.py @ 18484:d43823f928fe stable
largefiles: adapt remotestore._getfile to batched statlfile
9e1616307c4c introduced batching of statlfile, but not all codepaths got
converted.
_getfile gave _stat garbage and got garbage back. The garbage didn't match the
expected error codes and was thus interpreted as success. It could thus end up
trying to fetch a largefile that didn't exist.
Instead we now pass _stat valid input and handle both correct and invalid
output correctly.
This makes the code work as intended ... but it would probably be better if it
didn't abort on missing largefiles, just like it happened to do before.
author | Mads Kiilerich <madski@unity3d.com> |
---|---|
date | Mon, 28 Jan 2013 15:19:44 +0100 |
parents | 6f219eb83435 |
children | 003730ca254d |
rev | line source |
---|---|
15168 | 1 # Copyright 2010-2011 Fog Creek Software |
2 # Copyright 2010-2011 Unity Technologies | |
3 # | |
4 # This software may be used and distributed according to the terms of the | |
5 # GNU General Public License version 2 or any later version. | |
6 | |
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
7 '''remote largefile store; the base class for wirestore''' |
15168 | 8 |
9 import urllib2 | |
10 | |
11 from mercurial import util | |
12 from mercurial.i18n import _ | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
13 from mercurial.wireproto import remotebatch |
15168 | 14 |
15 import lfutil | |
16 import basestore | |
17 | |
18 class remotestore(basestore.basestore): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15188
diff
changeset
|
19 '''a largefile store accessed over a network''' |
15168 | 20 def __init__(self, ui, repo, url): |
21 super(remotestore, self).__init__(ui, repo, url) | |
22 | |
23 def put(self, source, hash): | |
24 if self.sendfile(source, hash): | |
25 raise util.Abort( | |
26 _('remotestore: could not put %s to remote store %s') | |
27 % (source, self.url)) | |
28 self.ui.debug( | |
29 _('remotestore: put %s to remote store %s') % (source, self.url)) | |
30 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
31 def exists(self, hashes): |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
32 return self._verify(hashes) |
15168 | 33 |
34 def sendfile(self, filename, hash): | |
35 self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash)) | |
36 fd = None | |
37 try: | |
38 try: | |
39 fd = lfutil.httpsendfile(self.ui, filename) | |
40 except IOError, e: | |
41 raise util.Abort( | |
42 _('remotestore: could not open file %s: %s') | |
43 % (filename, str(e))) | |
44 return self._put(hash, fd) | |
45 finally: | |
46 if fd: | |
47 fd.close() | |
48 | |
49 def _getfile(self, tmpfile, filename, hash): | |
50 # quit if the largefile isn't there | |
18484
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
51 stat = self._stat([hash])[hash] |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
52 if stat == 1: |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
53 raise util.Abort(_('remotestore: largefile %s is invalid') % hash) |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
54 elif stat == 2: |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
55 raise util.Abort(_('remotestore: largefile %s is missing') % hash) |
18484
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
56 elif stat != 0: |
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
57 raise RuntimeError('error getting file: unexpected response from ' |
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
58 'statlfile (%r)' % stat) |
15168 | 59 |
60 try: | |
61 length, infile = self._get(hash) | |
15188
8e115063950d
largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents:
15168
diff
changeset
|
62 except urllib2.HTTPError, e: |
15168 | 63 # 401s get converted to util.Aborts; everything else is fine being |
64 # turned into a StoreError | |
65 raise basestore.StoreError(filename, hash, self.url, str(e)) | |
66 except urllib2.URLError, e: | |
67 # This usually indicates a connection problem, so don't | |
68 # keep trying with the other files... they will probably | |
69 # all fail too. | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
70 raise util.Abort('%s: %s' % (self.url, e.reason)) |
15168 | 71 except IOError, e: |
72 raise basestore.StoreError(filename, hash, self.url, str(e)) | |
73 | |
74 # Mercurial does not close its SSH connections after writing a stream | |
75 if length is not None: | |
76 infile = lfutil.limitreader(infile, length) | |
77 return lfutil.copyandhash(lfutil.blockstream(infile), tmpfile) | |
78 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
79 def _verify(self, hashes): |
18481
ed647c59753b
largefiles: let wirestore._stat return stats as expected by remotestore verify
Mads Kiilerich <madski@unity3d.com>
parents:
17425
diff
changeset
|
80 return dict((h, s == 0) for (h, s) in self._stat(hashes).iteritems()) |
15168 | 81 |
82 def _verifyfile(self, cctx, cset, contents, standin, verified): | |
83 filename = lfutil.splitstandin(standin) | |
84 if not filename: | |
85 return False | |
86 fctx = cctx[standin] | |
87 key = (filename, fctx.filenode()) | |
88 if key in verified: | |
89 return False | |
90 | |
91 verified.add(key) | |
92 | |
18482
6f219eb83435
largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents:
18481
diff
changeset
|
93 expecthash = fctx.data()[0:40] |
6f219eb83435
largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents:
18481
diff
changeset
|
94 stat = self._stat([expecthash])[expecthash] |
15168 | 95 if not stat: |
96 return False | |
97 elif stat == 1: | |
98 self.ui.warn( | |
99 _('changeset %s: %s: contents differ\n') | |
100 % (cset, filename)) | |
101 return True # failed | |
102 elif stat == 2: | |
103 self.ui.warn( | |
104 _('changeset %s: %s missing\n') | |
105 % (cset, filename)) | |
106 return True # failed | |
107 else: | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
108 raise RuntimeError('verify failed: unexpected response from ' |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
109 'statlfile (%r)' % stat) |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
110 |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
111 def batch(self): |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
112 '''Support for remote batching.''' |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
113 return remotebatch(self) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
114 |