Mercurial > hg
annotate hgext/largefiles/remotestore.py @ 29218:fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
Files that are already in local store should be checked locally. The problem
with this implementation is how difference in messages between local and remote
checks should look like. For now local errors for file missing and content
corrupted looks like this:
'changeset cset: filename references missing storepath\n'
'changeset cset: filename references corrupted storepath\n'
for remote it looks like:
'changeset cset: filename missing\n'
'changeset cset: filename: contents differ\n'
Contents differ error for remote calls is never raised currently - for now
statlfile implementation lacks checking file content.
author | liscju <piotr.listkiewicz@gmail.com> |
---|---|
date | Mon, 09 May 2016 10:05:32 +0200 |
parents | 305f9c36a0f5 |
children | 0ccab84f9630 |
rev | line source |
---|---|
15168 | 1 # Copyright 2010-2011 Fog Creek Software |
2 # Copyright 2010-2011 Unity Technologies | |
3 # | |
4 # This software may be used and distributed according to the terms of the | |
5 # GNU General Public License version 2 or any later version. | |
6 | |
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
7 '''remote largefile store; the base class for wirestore''' |
15168 | 8 |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
9 from mercurial import util, wireproto, error |
15168 | 10 from mercurial.i18n import _ |
11 | |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
12 urlerr = util.urlerr |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
13 urlreq = util.urlreq |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
14 |
15168 | 15 import lfutil |
16 import basestore | |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
17 import localstore |
15168 | 18 |
19 class remotestore(basestore.basestore): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15188
diff
changeset
|
20 '''a largefile store accessed over a network''' |
15168 | 21 def __init__(self, ui, repo, url): |
22 super(remotestore, self).__init__(ui, repo, url) | |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
23 self._lstore = localstore.localstore(self.ui, self.repo, self.repo) |
15168 | 24 |
25 def put(self, source, hash): | |
26 if self.sendfile(source, hash): | |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
27 raise error.Abort( |
15168 | 28 _('remotestore: could not put %s to remote store %s') |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
29 % (source, util.hidepassword(self.url))) |
15168 | 30 self.ui.debug( |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
31 _('remotestore: put %s to remote store %s\n') |
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
32 % (source, util.hidepassword(self.url))) |
15168 | 33 |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
34 def exists(self, hashes): |
20688
a61ed1c2d7a7
check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents:
19950
diff
changeset
|
35 return dict((h, s == 0) for (h, s) in # dict-from-generator |
a61ed1c2d7a7
check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents:
19950
diff
changeset
|
36 self._stat(hashes).iteritems()) |
15168 | 37 |
38 def sendfile(self, filename, hash): | |
39 self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash)) | |
40 fd = None | |
41 try: | |
25079
bee00e0c2e45
largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
21084
diff
changeset
|
42 fd = lfutil.httpsendfile(self.ui, filename) |
15168 | 43 return self._put(hash, fd) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25079
diff
changeset
|
44 except IOError as e: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
45 raise error.Abort( |
25079
bee00e0c2e45
largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
21084
diff
changeset
|
46 _('remotestore: could not open file %s: %s') |
bee00e0c2e45
largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
21084
diff
changeset
|
47 % (filename, str(e))) |
15168 | 48 finally: |
49 if fd: | |
50 fd.close() | |
51 | |
52 def _getfile(self, tmpfile, filename, hash): | |
53 try: | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
54 chunks = self._get(hash) |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
55 except urlerr.httperror as e: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
56 # 401s get converted to error.Aborts; everything else is fine being |
15168 | 57 # turned into a StoreError |
58 raise basestore.StoreError(filename, hash, self.url, str(e)) | |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
59 except urlerr.urlerror as e: |
15168 | 60 # This usually indicates a connection problem, so don't |
61 # keep trying with the other files... they will probably | |
62 # all fail too. | |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
63 raise error.Abort('%s: %s' % |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
64 (util.hidepassword(self.url), e.reason)) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25079
diff
changeset
|
65 except IOError as e: |
15168 | 66 raise basestore.StoreError(filename, hash, self.url, str(e)) |
67 | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
68 return lfutil.copyandhash(chunks, tmpfile) |
15168 | 69 |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
70 def _hashesavailablelocally(self, hashes): |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
71 existslocallymap = self._lstore.exists(hashes) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
72 localhashes = [hash for hash in hashes if existslocallymap[hash]] |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
73 return localhashes |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
74 |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
75 def _verifyfiles(self, contents, filestocheck): |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
76 failed = False |
29068
305f9c36a0f5
largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents:
29067
diff
changeset
|
77 expectedhashes = [expectedhash |
305f9c36a0f5
largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents:
29067
diff
changeset
|
78 for cset, filename, expectedhash in filestocheck] |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
79 localhashes = self._hashesavailablelocally(expectedhashes) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
80 stats = self._stat([expectedhash for expectedhash in expectedhashes |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
81 if expectedhash not in localhashes]) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
82 |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
83 for cset, filename, expectedhash in filestocheck: |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
84 if expectedhash in localhashes: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
85 filetocheck = (cset, filename, expectedhash) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
86 verifyresult = self._lstore._verifyfiles(contents, |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
87 [filetocheck]) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
88 if verifyresult: |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
89 failed = True |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
90 else: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
91 stat = stats[expectedhash] |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
92 if stat: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
93 if stat == 1: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
94 self.ui.warn( |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
95 _('changeset %s: %s: contents differ\n') |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
96 % (cset, filename)) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
97 failed = True |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
98 elif stat == 2: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
99 self.ui.warn( |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
100 _('changeset %s: %s missing\n') |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
101 % (cset, filename)) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
102 failed = True |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
103 else: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
104 raise RuntimeError('verify failed: unexpected response ' |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
105 'from statlfile (%r)' % stat) |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
106 return failed |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
107 |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
108 def batch(self): |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
109 '''Support for remote batching.''' |
21084
70252bdfd39c
largefiles: import whole modules instead of importing parts of them
Mads Kiilerich <madski@unity3d.com>
parents:
20688
diff
changeset
|
110 return wireproto.remotebatch(self) |
28442
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
111 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
112 def _put(self, hash, fd): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
113 '''Put file with the given hash in the remote store.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
114 raise NotImplementedError('abstract method') |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
115 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
116 def _get(self, hash): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
117 '''Get file with the given hash from the remote store.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
118 raise NotImplementedError('abstract method') |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
119 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
120 def _stat(self, hashes): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
121 '''Get information about availability of files specified by |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
122 hashes in the remote store. Return dictionary mapping hashes |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
123 to return code where 0 means that file is available, other |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
124 values if not.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
125 raise NotImplementedError('abstract method') |