annotate hgext/largefiles/storefactory.py @ 35564:cf841f2b5a72

largefiles: add support for 'largefiles://' url scheme This changesets allows Mercurial to transparently download content from the configured largefile store. This handle all authentication and largefile protocol details. The target usecase is to leverage largefile infrastructure for clone bundle. See next changeset for details
author Boris Feld <boris.feld@octobus.net>
date Thu, 21 Dec 2017 13:57:57 +0100
parents 85868ecf2c0d
children ea62c2df882d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
1 # This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
2 # GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
3
29305
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
4 from __future__ import absolute_import
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
5
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
6 import re
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
7
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
8 from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
9
29305
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
10 from mercurial import (
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
11 error,
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
12 hg,
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
13 util,
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
14 )
19918
ae65192fd6b4 largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents: 19008
diff changeset
15
29305
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
16 from . import (
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
17 lfutil,
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
18 localstore,
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
19 wirestore,
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
20 )
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
21
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
22 # During clone this function is passed the src's ui object
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
23 # but it needs the dest's ui object so it can read out of
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
24 # the config file. Use repo.ui instead.
35564
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
25 def openstore(repo=None, remote=None, put=False, ui=None):
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
26 if ui is None:
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
27 ui = repo.ui
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
28
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
29 if not remote:
15943
f9efb325ea32 largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15319
diff changeset
30 lfpullsource = getattr(repo, 'lfpullsource', None)
f9efb325ea32 largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15319
diff changeset
31 if lfpullsource:
f9efb325ea32 largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15319
diff changeset
32 path = ui.expandpath(lfpullsource)
23941
164bd5218ddb largefiles: use 'default' path for pulling largefiles, not 'default-push'
Mads Kiilerich <madski@unity3d.com>
parents: 19950
diff changeset
33 elif put:
164bd5218ddb largefiles: use 'default' path for pulling largefiles, not 'default-push'
Mads Kiilerich <madski@unity3d.com>
parents: 19950
diff changeset
34 path = ui.expandpath('default-push', 'default')
15943
f9efb325ea32 largefiles: fix caching largefiles from an aliased repo (issue3212)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15319
diff changeset
35 else:
23941
164bd5218ddb largefiles: use 'default' path for pulling largefiles, not 'default-push'
Mads Kiilerich <madski@unity3d.com>
parents: 19950
diff changeset
36 path = ui.expandpath('default')
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15169
diff changeset
37
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15169
diff changeset
38 # ui.expandpath() leaves 'default-push' and 'default' alone if
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15169
diff changeset
39 # they cannot be expanded: fallback to the empty string,
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15169
diff changeset
40 # meaning the current directory.
35564
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
41 if repo is None:
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
42 path = ui.expandpath('default')
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
43 path, _branches = hg.parseurl(path)
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
44 remote = hg.peer(repo or ui, {}, path)
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
45 elif path == 'default-push' or path == 'default':
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
46 path = ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
47 remote = repo
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
48 else:
18489
f1700480bef7 largefiles: allow use of urls with #revision
Mads Kiilerich <madski@unity3d.com>
parents: 18486
diff changeset
49 path, _branches = hg.parseurl(path)
35564
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
50 remote = hg.peer(repo or ui, {}, path)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
51
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
52 # The path could be a scheme so use Mercurial's normal functionality
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
53 # to resolve the scheme to a repository and use its path
15169
aa262fff87ac largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents: 15168
diff changeset
54 path = util.safehasattr(remote, 'url') and remote.url() or remote.path
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
55
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
56 match = _scheme_re.match(path)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
57 if not match: # regular filesystem path
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
58 scheme = 'file'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
59 else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
60 scheme = match.group(1)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
61
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
62 try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
63 storeproviders = _storeprovider[scheme]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
64 except KeyError:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
65 raise error.Abort(_('unsupported URL scheme %r') % scheme)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
66
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16154
diff changeset
67 for classobj in storeproviders:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
68 try:
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16154
diff changeset
69 return classobj(ui, repo, remote)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
70 except lfutil.storeprotonotcapable:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
71 pass
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
72
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
73 raise error.Abort(_('%s does not appear to be a largefile store') %
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19918
diff changeset
74 util.hidepassword(path))
29305
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
75
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
76 _storeprovider = {
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
77 'file': [localstore.localstore],
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
78 'http': [wirestore.wirestore],
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
79 'https': [wirestore.wirestore],
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
80 'ssh': [wirestore.wirestore],
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
81 }
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
82
814076f4ace3 largefiles: move basestore._openstore into new module to remove cycle
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
83 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
35564
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
84
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
85 def getlfile(ui, hash):
cf841f2b5a72 largefiles: add support for 'largefiles://' url scheme
Boris Feld <boris.feld@octobus.net>
parents: 29355
diff changeset
86 return util.chunkbuffer(openstore(ui=ui)._get(hash))