annotate hgext/largefiles/remotestore.py @ 36016:ca62987f3f7d

py3: replace file() with open() in test-transplant.t file() is not present in Python 3. This patch also adds a b'' prefix to make sure we write bytes in Python 3. Differential Revision: https://phab.mercurial-scm.org/D2115
author Pulkit Goyal <7895pulkit@gmail.com>
date Sun, 11 Feb 2018 17:00:21 +0530
parents cf841f2b5a72
children 247e9bf4ecdc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
1 # Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
2 # Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
3 #
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
4 # This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
5 # GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
6
17425
e95ec38f86b0 fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents: 17127
diff changeset
7 '''remote largefile store; the base class for wirestore'''
29313
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
8 from __future__ import absolute_import
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
9
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
10 from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
11
29313
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
12 from mercurial import (
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
13 error,
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
14 util,
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
15 )
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
16
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
17 from . import (
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
18 basestore,
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
19 lfutil,
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
20 localstore,
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
21 )
0ccab84f9630 py3: make largefiles/remotestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 29218
diff changeset
22
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
23 urlerr = util.urlerr
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
24 urlreq = util.urlreq
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
25
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
26 class remotestore(basestore.basestore):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15188
diff changeset
27 '''a largefile store accessed over a network'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
28 def __init__(self, ui, repo, url):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
29 super(remotestore, self).__init__(ui, repo, url)
35564
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 33763
diff changeset
30 self._lstore = None
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 33763
diff changeset
31 if repo is not None:
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 33763
diff changeset
32 self._lstore = localstore.localstore(self.ui, self.repo, self.repo)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
33
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
34 def put(self, source, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
35 if self.sendfile(source, hash):
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
36 raise error.Abort(
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
37 _('remotestore: could not put %s to remote store %s')
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
38 % (source, util.hidepassword(self.url)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
39 self.ui.debug(
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
40 _('remotestore: put %s to remote store %s\n')
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
41 % (source, util.hidepassword(self.url)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
42
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
43 def exists(self, hashes):
20688
a61ed1c2d7a7 check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents: 19950
diff changeset
44 return dict((h, s == 0) for (h, s) in # dict-from-generator
a61ed1c2d7a7 check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents: 19950
diff changeset
45 self._stat(hashes).iteritems())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
46
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
47 def sendfile(self, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
48 self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
49 try:
30142
3dcaf1c4e90d largefiles: use context for file closing
Mads Kiilerich <madski@unity3d.com>
parents: 29313
diff changeset
50 with lfutil.httpsendfile(self.ui, filename) as fd:
3dcaf1c4e90d largefiles: use context for file closing
Mads Kiilerich <madski@unity3d.com>
parents: 29313
diff changeset
51 return self._put(hash, fd)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
52 except IOError as e:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
53 raise error.Abort(
25079
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
54 _('remotestore: could not open file %s: %s')
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
55 % (filename, str(e)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
56
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
57 def _getfile(self, tmpfile, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
58 try:
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
59 chunks = self._get(hash)
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
60 except urlerr.httperror as e:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
61 # 401s get converted to error.Aborts; everything else is fine being
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
62 # turned into a StoreError
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
63 raise basestore.StoreError(filename, hash, self.url, str(e))
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
64 except urlerr.urlerror as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
65 # This usually indicates a connection problem, so don't
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
66 # keep trying with the other files... they will probably
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
67 # all fail too.
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
68 raise error.Abort('%s: %s' %
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
69 (util.hidepassword(self.url), e.reason))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
70 except IOError as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
71 raise basestore.StoreError(filename, hash, self.url, str(e))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
72
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
73 return lfutil.copyandhash(chunks, tmpfile)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
74
29218
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
75 def _hashesavailablelocally(self, hashes):
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
76 existslocallymap = self._lstore.exists(hashes)
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
77 localhashes = [hash for hash in hashes if existslocallymap[hash]]
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
78 return localhashes
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
79
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
80 def _verifyfiles(self, contents, filestocheck):
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
81 failed = False
29068
305f9c36a0f5 largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
82 expectedhashes = [expectedhash
305f9c36a0f5 largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
83 for cset, filename, expectedhash in filestocheck]
29218
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
84 localhashes = self._hashesavailablelocally(expectedhashes)
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
85 stats = self._stat([expectedhash for expectedhash in expectedhashes
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
86 if expectedhash not in localhashes])
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
87
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
88 for cset, filename, expectedhash in filestocheck:
29218
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
89 if expectedhash in localhashes:
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
90 filetocheck = (cset, filename, expectedhash)
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
91 verifyresult = self._lstore._verifyfiles(contents,
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
92 [filetocheck])
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
93 if verifyresult:
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
94 failed = True
29218
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
95 else:
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
96 stat = stats[expectedhash]
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
97 if stat:
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
98 if stat == 1:
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
99 self.ui.warn(
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
100 _('changeset %s: %s: contents differ\n')
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
101 % (cset, filename))
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
102 failed = True
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
103 elif stat == 2:
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
104 self.ui.warn(
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
105 _('changeset %s: %s missing\n')
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
106 % (cset, filename))
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
107 failed = True
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
108 else:
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
109 raise RuntimeError('verify failed: unexpected response '
fd288d118074 largefiles: send statlfile remote calls only for nonexisting locally files
liscju <piotr.listkiewicz@gmail.com>
parents: 29068
diff changeset
110 'from statlfile (%r)' % stat)
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
111 return failed
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
112
28442
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
113 def _put(self, hash, fd):
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
114 '''Put file with the given hash in the remote store.'''
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
115 raise NotImplementedError('abstract method')
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
116
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
117 def _get(self, hash):
30180
736f92c44656 largefiles: always use filechunkiter when iterating files
Mads Kiilerich <madski@unity3d.com>
parents: 30142
diff changeset
118 '''Get a iterator for content with the given hash.'''
28442
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
119 raise NotImplementedError('abstract method')
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
120
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
121 def _stat(self, hashes):
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
122 '''Get information about availability of files specified by
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
123 hashes in the remote store. Return dictionary mapping hashes
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
124 to return code where 0 means that file is available, other
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
125 values if not.'''
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
126 raise NotImplementedError('abstract method')