Mercurial > hg
annotate hgext/largefiles/remotestore.py @ 36562:247e9bf4ecdc
py3: use util.forcebytestr() to convert IOErrors to bytes
Differential Revision: https://phab.mercurial-scm.org/D2536
author | Pulkit Goyal <7895pulkit@gmail.com> |
---|---|
date | Fri, 02 Mar 2018 07:14:59 +0530 |
parents | cf841f2b5a72 |
children | f0b6fbea00cf |
rev | line source |
---|---|
15168 | 1 # Copyright 2010-2011 Fog Creek Software |
2 # Copyright 2010-2011 Unity Technologies | |
3 # | |
4 # This software may be used and distributed according to the terms of the | |
5 # GNU General Public License version 2 or any later version. | |
6 | |
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
7 '''remote largefile store; the base class for wirestore''' |
29313
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
8 from __future__ import absolute_import |
15168 | 9 |
10 from mercurial.i18n import _ | |
11 | |
29313
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
12 from mercurial import ( |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
13 error, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
14 util, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
15 ) |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
16 |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
17 from . import ( |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
18 basestore, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
19 lfutil, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
20 localstore, |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
21 ) |
0ccab84f9630
py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29218
diff
changeset
|
22 |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
23 urlerr = util.urlerr |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
24 urlreq = util.urlreq |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
25 |
15168 | 26 class remotestore(basestore.basestore): |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15188
diff
changeset
|
27 '''a largefile store accessed over a network''' |
15168 | 28 def __init__(self, ui, repo, url): |
29 super(remotestore, self).__init__(ui, repo, url) | |
35564
cf841f2b5a72
largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents:
33763
diff
changeset
|
30 self._lstore = None |
cf841f2b5a72
largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents:
33763
diff
changeset
|
31 if repo is not None: |
cf841f2b5a72
largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents:
33763
diff
changeset
|
32 self._lstore = localstore.localstore(self.ui, self.repo, self.repo) |
15168 | 33 |
34 def put(self, source, hash): | |
35 if self.sendfile(source, hash): | |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
36 raise error.Abort( |
15168 | 37 _('remotestore: could not put %s to remote store %s') |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
38 % (source, util.hidepassword(self.url))) |
15168 | 39 self.ui.debug( |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
40 _('remotestore: put %s to remote store %s\n') |
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
41 % (source, util.hidepassword(self.url))) |
15168 | 42 |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
43 def exists(self, hashes): |
20688
a61ed1c2d7a7
check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents:
19950
diff
changeset
|
44 return dict((h, s == 0) for (h, s) in # dict-from-generator |
a61ed1c2d7a7
check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents:
19950
diff
changeset
|
45 self._stat(hashes).iteritems()) |
15168 | 46 |
47 def sendfile(self, filename, hash): | |
48 self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash)) | |
49 try: | |
30142
3dcaf1c4e90d
largefiles: use context for file closing
Mads Kiilerich <madski@unity3d.com>
parents:
29313
diff
changeset
|
50 with lfutil.httpsendfile(self.ui, filename) as fd: |
3dcaf1c4e90d
largefiles: use context for file closing
Mads Kiilerich <madski@unity3d.com>
parents:
29313
diff
changeset
|
51 return self._put(hash, fd) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25079
diff
changeset
|
52 except IOError as e: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
53 raise error.Abort( |
25079
bee00e0c2e45
largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
21084
diff
changeset
|
54 _('remotestore: could not open file %s: %s') |
36562
247e9bf4ecdc
py3: use util.forcebytestr() to convert IOErrors to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35564
diff
changeset
|
55 % (filename, util.forcebytestr(e))) |
15168 | 56 |
57 def _getfile(self, tmpfile, filename, hash): | |
58 try: | |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
59 chunks = self._get(hash) |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
60 except urlerr.httperror as e: |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
61 # 401s get converted to error.Aborts; everything else is fine being |
15168 | 62 # turned into a StoreError |
36562
247e9bf4ecdc
py3: use util.forcebytestr() to convert IOErrors to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35564
diff
changeset
|
63 raise basestore.StoreError(filename, hash, self.url, |
247e9bf4ecdc
py3: use util.forcebytestr() to convert IOErrors to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35564
diff
changeset
|
64 util.forcebytestr(e)) |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28442
diff
changeset
|
65 except urlerr.urlerror as e: |
15168 | 66 # This usually indicates a connection problem, so don't |
67 # keep trying with the other files... they will probably | |
68 # all fail too. | |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25660
diff
changeset
|
69 raise error.Abort('%s: %s' % |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19948
diff
changeset
|
70 (util.hidepassword(self.url), e.reason)) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25079
diff
changeset
|
71 except IOError as e: |
36562
247e9bf4ecdc
py3: use util.forcebytestr() to convert IOErrors to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35564
diff
changeset
|
72 raise basestore.StoreError(filename, hash, self.url, |
247e9bf4ecdc
py3: use util.forcebytestr() to convert IOErrors to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35564
diff
changeset
|
73 util.forcebytestr(e)) |
15168 | 74 |
19004
6614e5e24e66
largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
75 return lfutil.copyandhash(chunks, tmpfile) |
15168 | 76 |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
77 def _hashesavailablelocally(self, hashes): |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
78 existslocallymap = self._lstore.exists(hashes) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
79 localhashes = [hash for hash in hashes if existslocallymap[hash]] |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
80 return localhashes |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
81 |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
82 def _verifyfiles(self, contents, filestocheck): |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
83 failed = False |
29068
305f9c36a0f5
largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents:
29067
diff
changeset
|
84 expectedhashes = [expectedhash |
305f9c36a0f5
largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents:
29067
diff
changeset
|
85 for cset, filename, expectedhash in filestocheck] |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
86 localhashes = self._hashesavailablelocally(expectedhashes) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
87 stats = self._stat([expectedhash for expectedhash in expectedhashes |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
88 if expectedhash not in localhashes]) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
89 |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
90 for cset, filename, expectedhash in filestocheck: |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
91 if expectedhash in localhashes: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
92 filetocheck = (cset, filename, expectedhash) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
93 verifyresult = self._lstore._verifyfiles(contents, |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
94 [filetocheck]) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
95 if verifyresult: |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
96 failed = True |
29218
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
97 else: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
98 stat = stats[expectedhash] |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
99 if stat: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
100 if stat == 1: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
101 self.ui.warn( |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
102 _('changeset %s: %s: contents differ\n') |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
103 % (cset, filename)) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
104 failed = True |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
105 elif stat == 2: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
106 self.ui.warn( |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
107 _('changeset %s: %s missing\n') |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
108 % (cset, filename)) |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
109 failed = True |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
110 else: |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
111 raise RuntimeError('verify failed: unexpected response ' |
fd288d118074
largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents:
29068
diff
changeset
|
112 'from statlfile (%r)' % stat) |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28883
diff
changeset
|
113 return failed |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
15253
diff
changeset
|
114 |
28442
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
115 def _put(self, hash, fd): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
116 '''Put file with the given hash in the remote store.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
117 raise NotImplementedError('abstract method') |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
118 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
119 def _get(self, hash): |
30180
736f92c44656
largefiles: always use filechunkiter when iterating files
Mads Kiilerich <madski@unity3d.com>
parents:
30142
diff
changeset
|
120 '''Get a iterator for content with the given hash.''' |
28442
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
121 raise NotImplementedError('abstract method') |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
122 |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
123 def _stat(self, hashes): |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
124 '''Get information about availability of files specified by |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
125 hashes in the remote store. Return dictionary mapping hashes |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
126 to return code where 0 means that file is available, other |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
127 values if not.''' |
3be2e89c5d9f
largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents:
26587
diff
changeset
|
128 raise NotImplementedError('abstract method') |