author | Mads Kiilerich <madski@unity3d.com> |
Mon, 15 Apr 2013 23:43:50 +0200 | |
changeset 19002 | 5083baa6cbf8 |
parent 19001 | 2a35296a6304 |
child 19003 | ad993cb7bbb1 |
permissions | -rw-r--r-- |
15168 | 1 |
# Copyright 2010-2011 Fog Creek Software |
2 |
# Copyright 2010-2011 Unity Technologies |
|
3 |
# |
|
4 |
# This software may be used and distributed according to the terms of the |
|
5 |
# GNU General Public License version 2 or any later version. |
|
6 |
||
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
7 |
'''remote largefile store; the base class for wirestore''' |
15168 | 8 |
|
9 |
import urllib2 |
|
10 |
||
11 |
from mercurial import util |
|
12 |
from mercurial.i18n import _ |
|
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
13 |
from mercurial.wireproto import remotebatch |
15168 | 14 |
|
15 |
import lfutil |
|
16 |
import basestore |
|
17 |
||
18 |
class remotestore(basestore.basestore): |
|
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15188
diff
changeset
|
19 |
'''a largefile store accessed over a network''' |
15168 | 20 |
def __init__(self, ui, repo, url): |
21 |
super(remotestore, self).__init__(ui, repo, url) |
|
22 |
||
23 |
def put(self, source, hash): |
|
24 |
if self.sendfile(source, hash): |
|
25 |
raise util.Abort( |
|
26 |
_('remotestore: could not put %s to remote store %s') |
|
27 |
% (source, self.url)) |
|
28 |
self.ui.debug( |
|
29 |
_('remotestore: put %s to remote store %s') % (source, self.url)) |
|
30 |
||
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
31 |
def exists(self, hashes): |
18573
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18484
diff
changeset
|
32 |
return dict((h, s == 0) for (h, s) in self._stat(hashes).iteritems()) |
15168 | 33 |
|
34 |
def sendfile(self, filename, hash): |
|
35 |
self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash)) |
|
36 |
fd = None |
|
37 |
try: |
|
38 |
try: |
|
39 |
fd = lfutil.httpsendfile(self.ui, filename) |
|
40 |
except IOError, e: |
|
41 |
raise util.Abort( |
|
42 |
_('remotestore: could not open file %s: %s') |
|
43 |
% (filename, str(e))) |
|
44 |
return self._put(hash, fd) |
|
45 |
finally: |
|
46 |
if fd: |
|
47 |
fd.close() |
|
48 |
||
49 |
def _getfile(self, tmpfile, filename, hash): |
|
50 |
# quit if the largefile isn't there |
|
18484
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
51 |
stat = self._stat([hash])[hash] |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
52 |
if stat == 1: |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
53 |
raise util.Abort(_('remotestore: largefile %s is invalid') % hash) |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
54 |
elif stat == 2: |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
55 |
raise util.Abort(_('remotestore: largefile %s is missing') % hash) |
18484
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
56 |
elif stat != 0: |
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
57 |
raise RuntimeError('error getting file: unexpected response from ' |
d43823f928fe
largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents:
18482
diff
changeset
|
58 |
'statlfile (%r)' % stat) |
15168 | 59 |
|
60 |
try: |
|
61 |
length, infile = self._get(hash) |
|
15188
8e115063950d
largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents:
15168
diff
changeset
|
62 |
except urllib2.HTTPError, e: |
15168 | 63 |
# 401s get converted to util.Aborts; everything else is fine being |
64 |
# turned into a StoreError |
|
65 |
raise basestore.StoreError(filename, hash, self.url, str(e)) |
|
66 |
except urllib2.URLError, e: |
|
67 |
# This usually indicates a connection problem, so don't |
|
68 |
# keep trying with the other files... they will probably |
|
69 |
# all fail too. |
|
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
70 |
raise util.Abort('%s: %s' % (self.url, e.reason)) |
15168 | 71 |
except IOError, e: |
72 |
raise basestore.StoreError(filename, hash, self.url, str(e)) |
|
73 |
||
74 |
# Mercurial does not close its SSH connections after writing a stream |
|
75 |
if length is not None: |
|
76 |
infile = lfutil.limitreader(infile, length) |
|
19001
2a35296a6304
largefiles: drop lfutil.blockstream - use filechunkiter like everybody else
Mads Kiilerich <madski@unity3d.com>
parents:
18573
diff
changeset
|
77 |
try: |
2a35296a6304
largefiles: drop lfutil.blockstream - use filechunkiter like everybody else
Mads Kiilerich <madski@unity3d.com>
parents:
18573
diff
changeset
|
78 |
return lfutil.copyandhash(util.filechunkiter(infile, 128 * 1024), |
2a35296a6304
largefiles: drop lfutil.blockstream - use filechunkiter like everybody else
Mads Kiilerich <madski@unity3d.com>
parents:
18573
diff
changeset
|
79 |
tmpfile) |
2a35296a6304
largefiles: drop lfutil.blockstream - use filechunkiter like everybody else
Mads Kiilerich <madski@unity3d.com>
parents:
18573
diff
changeset
|
80 |
finally: |
2a35296a6304
largefiles: drop lfutil.blockstream - use filechunkiter like everybody else
Mads Kiilerich <madski@unity3d.com>
parents:
18573
diff
changeset
|
81 |
infile.close() |
19002
5083baa6cbf8
largefiles: remove blecch from lfutil.copyandhash - don't close the passed fd
Mads Kiilerich <madski@unity3d.com>
parents:
19001
diff
changeset
|
82 |
tmpfile.close() |
15168 | 83 |
|
84 |
def _verifyfile(self, cctx, cset, contents, standin, verified): |
|
85 |
filename = lfutil.splitstandin(standin) |
|
86 |
if not filename: |
|
87 |
return False |
|
88 |
fctx = cctx[standin] |
|
89 |
key = (filename, fctx.filenode()) |
|
90 |
if key in verified: |
|
91 |
return False |
|
92 |
||
93 |
verified.add(key) |
|
94 |
||
18482
6f219eb83435
largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents:
18481
diff
changeset
|
95 |
expecthash = fctx.data()[0:40] |
6f219eb83435
largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents:
18481
diff
changeset
|
96 |
stat = self._stat([expecthash])[expecthash] |
15168 | 97 |
if not stat: |
98 |
return False |
|
99 |
elif stat == 1: |
|
100 |
self.ui.warn( |
|
101 |
_('changeset %s: %s: contents differ\n') |
|
102 |
% (cset, filename)) |
|
103 |
return True # failed |
|
104 |
elif stat == 2: |
|
105 |
self.ui.warn( |
|
106 |
_('changeset %s: %s missing\n') |
|
107 |
% (cset, filename)) |
|
108 |
return True # failed |
|
109 |
else: |
|
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
110 |
raise RuntimeError('verify failed: unexpected response from ' |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
111 |
'statlfile (%r)' % stat) |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
112 |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
113 |
def batch(self): |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
114 |
'''Support for remote batching.''' |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
115 |
return remotebatch(self) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
116 |