annotate hgext/largefiles/localstore.py @ 25408:c88975a4d264

sshpeer: run the ssh command unbuffered This is necessary to use non-blocking IO base on polling. Such polling is needed to restore real time output with ssh peer. Changeset fce065538bcf is talking about 5x regression on Mac OS X when playing with this value. So we introduced our own buffering layer in previous changesets. This seems to keep the regression away (we are even issuing much less read).
author Pierre-Yves David <pierre-yves.david@fb.com>
date Wed, 20 May 2015 11:31:38 -0500
parents c082a4756ed7
children 40bd01be5c25
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
1 # Copyright 2009-2010 Gregory P. Ward
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
3 # Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
4 # Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
5 #
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
6 # This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
7 # GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
8
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15168
diff changeset
9 '''store class for local filesystem'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
10
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
11 from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
12
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
13 import lfutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
14 import basestore
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
15
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
16 class localstore(basestore.basestore):
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
17 '''localstore first attempts to grab files out of the store in the remote
17424
e7cfe3587ea4 fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents: 17191
diff changeset
18 Mercurial repository. Failing that, it attempts to grab the files from
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
19 the user cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
20
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
21 def __init__(self, ui, repo, remote):
17191
5884812686f7 peer: introduce peer methods to prepare for peer classes
Sune Foldager <cryo@cyanite.org>
parents: 16928
diff changeset
22 self.remote = remote.local()
18155
5206af8894a3 largefiles: cleanup of warnings on errors getting largefiles
Mads Kiilerich <madski@unity3d.com>
parents: 17439
diff changeset
23 super(localstore, self).__init__(ui, repo, self.remote.url())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
24
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
25 def put(self, source, hash):
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
26 if lfutil.instore(self.remote, hash):
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
27 return
19007
266b5fb72f26 largefiles: 'put' should store 'source' file in under 'hash', also in localstore
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
28 lfutil.link(source, lfutil.storepath(self.remote, hash))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
29
17411
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
30 def exists(self, hashes):
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
31 retval = {}
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
32 for hash in hashes:
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
33 retval[hash] = lfutil.instore(self.remote, hash)
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
34 return retval
a02e36568e88 largefiles: adjust localstore to handle batch statlfile requests (issue3583)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17191
diff changeset
35
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
36
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
37 def _getfile(self, tmpfile, filename, hash):
19000
eaf146e811a4 largefiles: refactoring - use findfile in localstore._getfile
Mads Kiilerich <madski@unity3d.com>
parents: 18998
diff changeset
38 path = lfutil.findfile(self.remote, hash)
eaf146e811a4 largefiles: refactoring - use findfile in localstore._getfile
Mads Kiilerich <madski@unity3d.com>
parents: 18998
diff changeset
39 if not path:
18155
5206af8894a3 largefiles: cleanup of warnings on errors getting largefiles
Mads Kiilerich <madski@unity3d.com>
parents: 17439
diff changeset
40 raise basestore.StoreError(filename, hash, self.url,
16928
73b9286e667c largefiles: lowercase messages
Martin Geisler <mg@aragost.com>
parents: 15371
diff changeset
41 _("can't get file locally"))
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
42 fd = open(path, 'rb')
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
43 try:
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
44 return lfutil.copyandhash(fd, tmpfile)
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
45 finally:
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
46 fd.close()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
47
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
48 def _verifyfile(self, cctx, cset, contents, standin, verified):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
49 filename = lfutil.splitstandin(standin)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
50 if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
51 return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
52 fctx = cctx[standin]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
53 key = (filename, fctx.filenode())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
54 if key in verified:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
55 return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
56
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
57 expecthash = fctx.data()[0:40]
24630
c082a4756ed7 largefiles: use lfutil.findstorepath() when verifying a local repo
Matt Harbison <matt_harbison@yahoo.com>
parents: 19007
diff changeset
58 storepath, exists = lfutil.findstorepath(self.remote, expecthash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
59 verified.add(key)
24630
c082a4756ed7 largefiles: use lfutil.findstorepath() when verifying a local repo
Matt Harbison <matt_harbison@yahoo.com>
parents: 19007
diff changeset
60 if not exists:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
61 self.ui.warn(
18545
a49b7c9fc246 largefiles: report localstore errors with single line warnings messages
Mads Kiilerich <madski@unity3d.com>
parents: 18155
diff changeset
62 _('changeset %s: %s references missing %s\n')
a49b7c9fc246 largefiles: report localstore errors with single line warnings messages
Mads Kiilerich <madski@unity3d.com>
parents: 18155
diff changeset
63 % (cset, filename, storepath))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
64 return True # failed
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
65
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
66 if contents:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
67 actualhash = lfutil.hashfile(storepath)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
68 if actualhash != expecthash:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
69 self.ui.warn(
18545
a49b7c9fc246 largefiles: report localstore errors with single line warnings messages
Mads Kiilerich <madski@unity3d.com>
parents: 18155
diff changeset
70 _('changeset %s: %s references corrupted %s\n')
a49b7c9fc246 largefiles: report localstore errors with single line warnings messages
Mads Kiilerich <madski@unity3d.com>
parents: 18155
diff changeset
71 % (cset, filename, storepath))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
72 return True # failed
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
73 return False