Mercurial > hg
annotate hgext/largefiles/remotestore.py @ 33763:dcdc17551653
largefiles: remove remotestore.batch()
This method was added in 9e1616307c4c. AFAICT it didn't do anything at
inception. If it did, there was no test coverage for it because
changing it to raise doesn't fail any tests at that revision.
b6e71f8af5b8 later refactored all remote.batch() calls to
remote.iterbatch(). So if this was somehow used, it isn't called
any more because there are no calls to .batch() remaining in the
repo.
I suspect the original patch author got confused by the distinction
between the peer/remote interface and the largefiles store. The lf
store is a gateway to a peer instance. It exposes additional
lf-specific methods to execute against a peer. However, it is not
a peer and doesn't need to implement batch() because peer itself
does that.
Differential Revision: https://phab.mercurial-scm.org/D316
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Wed, 09 Aug 2017 21:04:03 -0700 |
parents | 736f92c44656 |
children | cf841f2b5a72 |
rev | line source |
---|---|
15168 | 1 # Copyright 2010-2011 Fog Creek Software |
2 # Copyright 2010-2011 Unity Technologies | |
3 # | |
4 # This software may be used and distributed according to the terms of the | |
5 # GNU General Public License version 2 or any later version. | |
6 | |
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
7 '''remote largefile store; the base class for wirestore''' |
29313
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
8 from __future__ import absolute_import |
15168 | 9 |
10 from mercurial.i18n import _ | |
11 | |
29313
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
12 from mercurial import ( |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
13 error, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
14 util, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
15 ) |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
16 |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
17 from . import ( |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
18 basestore, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
19 lfutil, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
20 localstore, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
21 ) |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
22 |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
23 urlerr = util.urlerr |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
24 urlreq = util.urlreq |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
25 |
15168 | 26 class remotestore(basestore.basestore): |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15188
diff
changeset
|
27 '''a largefile store accessed over a network''' |
15168 | 28 def __init__(self, ui, repo, url): |
29 super(remotestore, self).__init__(ui, repo, url) | |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
30 self._lstore = localstore.localstore(self.ui, self.repo, self.repo) |
15168 | 31 |
32 def put(self, source, hash): | |
33 if self.sendfile(source, hash): | |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
34 raise error.Abort( |
15168 | 35 _('remotestore: could not put %s to remote store %s') |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
36 % (source, util.hidepassword(self.url))) |
15168 | 37 self.ui.debug( |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
38 _('remotestore: put %s to remote store %s\n') |
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
39 % (source, util.hidepassword(self.url))) |
15168 | 40 |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
41 def exists(self, hashes): |
20688
a61ed1c2d7a7
check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents:
19950
diff
changeset
|
42 return dict((h, s == 0) for (h, s) in # dict-from-generator |
a61ed1c2d7a7
check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents:
19950
diff
changeset
|
43 self._stat(hashes).iteritems()) |
15168 | 44 |
45 def sendfile(self, filename, hash): | |
46 self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash)) | |
47 try: | |
30142
3dcaf1c4e90d
largefiles: use context for file closing
Mads Kiilerich <madski@unity3d.com>
parents:
29313
diff
changeset
|
48 with lfutil.httpsendfile(self.ui, filename) as fd: |
3dcaf1c4e90d
largefiles: use context for file closing
Mads Kiilerich <madski@unity3d.com>
parents:
29313
diff
changeset
|
49 return self._put(hash, fd) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25079
diff
changeset
|
50 except IOError as e: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
51 raise error.Abort( |
25079
bee00e0c2e45
largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
21084
diff
changeset
|
52 _('remotestore: could not open file %s: %s') |
bee00e0c2e45
largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
21084
diff
changeset
|
53 % (filename, str(e))) |
15168 | 54 |
55 def _getfile(self, tmpfile, filename, hash): | |
56 try: | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
57 chunks = self._get(hash) |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
58 except urlerr.httperror as e: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
59 # 401s get converted to error.Aborts; everything else is fine being |
15168 | 60 # turned into a StoreError |
61 raise basestore.StoreError(filename, hash, self.url, str(e)) | |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
62 except urlerr.urlerror as e: |
15168 | 63 # This usually indicates a connection problem, so don't |
64 # keep trying with the other files... they will probably | |
65 # all fail too. | |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
66 raise error.Abort('%s: %s' % |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
67 (util.hidepassword(self.url), e.reason)) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25079
diff
changeset
|
68 except IOError as e: |
15168 | 69 raise basestore.StoreError(filename, hash, self.url, str(e)) |
70 | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
71 return lfutil.copyandhash(chunks, tmpfile) |
15168 | 72 |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
73 def _hashesavailablelocally(self, hashes): |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
74 existslocallymap = self._lstore.exists(hashes) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
75 localhashes = [hash for hash in hashes if existslocallymap[hash]] |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
76 return localhashes |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
77 |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
78 def _verifyfiles(self, contents, filestocheck): |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
79 failed = False |
29068
305f9c36a0f5
largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents:
29067
diff
changeset
|
80 expectedhashes = [expectedhash |
305f9c36a0f5
largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents:
29067
diff
changeset
|
81 for cset, filename, expectedhash in filestocheck] |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
82 localhashes = self._hashesavailablelocally(expectedhashes) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
83 stats = self._stat([expectedhash for expectedhash in expectedhashes |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
84 if expectedhash not in localhashes]) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
85 |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
86 for cset, filename, expectedhash in filestocheck: |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
87 if expectedhash in localhashes: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
88 filetocheck = (cset, filename, expectedhash) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
89 verifyresult = self._lstore._verifyfiles(contents, |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
90 [filetocheck]) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
91 if verifyresult: |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
92 failed = True |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
93 else: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
94 stat = stats[expectedhash] |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
95 if stat: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
96 if stat == 1: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
97 self.ui.warn( |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
98 _('changeset %s: %s: contents differ\n') |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
99 % (cset, filename)) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
100 failed = True |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
101 elif stat == 2: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
102 self.ui.warn( |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
103 _('changeset %s: %s missing\n') |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
104 % (cset, filename)) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
105 failed = True |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
106 else: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
107 raise RuntimeError('verify failed: unexpected response ' |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
108 'from statlfile (%r)' % stat) |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
109 return failed |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
110 |
28442
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
111 def _put(self, hash, fd): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
112 '''Put file with the given hash in the remote store.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
113 raise NotImplementedError('abstract method') |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
114 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
115 def _get(self, hash): |
30180
736f92c44656
largefiles: always use filechunkiter when iterating files
Mads Kiilerich <madski@unity3d.com>
parents:
30142
diff
changeset
|
116 '''Get a iterator for content with the given hash.''' |
28442
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
117 raise NotImplementedError('abstract method') |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
118 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
119 def _stat(self, hashes): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
120 '''Get information about availability of files specified by |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
121 hashes in the remote store. Return dictionary mapping hashes |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
122 to return code where 0 means that file is available, other |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
123 values if not.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
124 raise NotImplementedError('abstract method') |