Mercurial > hg
annotate hgext/largefiles/remotestore.py @ 19004:6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Avoid the intermediate limitreader and filechunkiter between getlfile and
copyandhash - return the right protocol and put the complexity where it better
can be managed.
author | Mads Kiilerich <madski@unity3d.com> |
---|---|
date | Tue, 16 Apr 2013 01:46:39 +0200 |
parents | ad993cb7bbb1 |
children | 9d33d6e0d442 |
rev | line source |
---|---|
15168 | 1 # Copyright 2010-2011 Fog Creek Software |
2 # Copyright 2010-2011 Unity Technologies | |
3 # | |
4 # This software may be used and distributed according to the terms of the | |
5 # GNU General Public License version 2 or any later version. | |
6 | |
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
7 '''remote largefile store; the base class for wirestore''' |
15168 | 8 |
9 import urllib2 | |
10 | |
11 from mercurial import util | |
12 from mercurial.i18n import _ | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
13 from mercurial.wireproto import remotebatch |
15168 | 14 |
15 import lfutil | |
16 import basestore | |
17 | |
18 class remotestore(basestore.basestore): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15188
diff
changeset
|
19 '''a largefile store accessed over a network''' |
15168 | 20 def __init__(self, ui, repo, url): |
21 super(remotestore, self).__init__(ui, repo, url) | |
22 | |
23 def put(self, source, hash): | |
24 if self.sendfile(source, hash): | |
25 raise util.Abort( | |
26 _('remotestore: could not put %s to remote store %s') | |
27 % (source, self.url)) | |
28 self.ui.debug( | |
29 _('remotestore: put %s to remote store %s') % (source, self.url)) | |
30 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
31 def exists(self, hashes): |
18573
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18484
diff
changeset
|
32 return dict((h, s == 0) for (h, s) in self._stat(hashes).iteritems()) |
15168 | 33 |
34 def sendfile(self, filename, hash): | |
35 self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash)) | |
36 fd = None | |
37 try: | |
38 try: | |
39 fd = lfutil.httpsendfile(self.ui, filename) | |
40 except IOError, e: | |
41 raise util.Abort( | |
42 _('remotestore: could not open file %s: %s') | |
43 % (filename, str(e))) | |
44 return self._put(hash, fd) | |
45 finally: | |
46 if fd: | |
47 fd.close() | |
48 | |
49 def _getfile(self, tmpfile, filename, hash): | |
50 # quit if the largefile isn't there | |
18484
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
51 stat = self._stat([hash])[hash] |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
52 if stat == 1: |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
53 raise util.Abort(_('remotestore: largefile %s is invalid') % hash) |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
54 elif stat == 2: |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
55 raise util.Abort(_('remotestore: largefile %s is missing') % hash) |
18484
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
56 elif stat != 0: |
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
57 raise RuntimeError('error getting file: unexpected response from ' |
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
58 'statlfile (%r)' % stat) |
15168 | 59 |
60 try: | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
61 chunks = self._get(hash) |
15188
8e115063950d
largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents:
15168
diff
changeset
|
62 except urllib2.HTTPError, e: |
15168 | 63 # 401s get converted to util.Aborts; everything else is fine being |
64 # turned into a StoreError | |
65 raise basestore.StoreError(filename, hash, self.url, str(e)) | |
66 except urllib2.URLError, e: | |
67 # This usually indicates a connection problem, so don't | |
68 # keep trying with the other files... they will probably | |
69 # all fail too. | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
70 raise util.Abort('%s: %s' % (self.url, e.reason)) |
15168 | 71 except IOError, e: |
72 raise basestore.StoreError(filename, hash, self.url, str(e)) | |
73 | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
74 return lfutil.copyandhash(chunks, tmpfile) |
15168 | 75 |
76 def _verifyfile(self, cctx, cset, contents, standin, verified): | |
77 filename = lfutil.splitstandin(standin) | |
78 if not filename: | |
79 return False | |
80 fctx = cctx[standin] | |
81 key = (filename, fctx.filenode()) | |
82 if key in verified: | |
83 return False | |
84 | |
85 verified.add(key) | |
86 | |
18482
6f219eb83435
largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents:
18481
diff
changeset
|
87 expecthash = fctx.data()[0:40] |
6f219eb83435
largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents:
18481
diff
changeset
|
88 stat = self._stat([expecthash])[expecthash] |
15168 | 89 if not stat: |
90 return False | |
91 elif stat == 1: | |
92 self.ui.warn( | |
93 _('changeset %s: %s: contents differ\n') | |
94 % (cset, filename)) | |
95 return True # failed | |
96 elif stat == 2: | |
97 self.ui.warn( | |
98 _('changeset %s: %s missing\n') | |
99 % (cset, filename)) | |
100 return True # failed | |
101 else: | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
102 raise RuntimeError('verify failed: unexpected response from ' |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
103 'statlfile (%r)' % stat) |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
104 |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
105 def batch(self): |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
106 '''Support for remote batching.''' |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
107 return remotebatch(self) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
108 |