Mercurial > hg
annotate hgext/largefiles/basestore.py @ 29484:53b7fc7cc2bb
sslutil: don't attempt to find default CA certs file when told not to
Before, devel.disableloaddefaultcerts only impacted the loading of
default certs via SSLContext. After this patch, the config option also
prevents sslutil._defaultcacerts() from being called.
This config option is meant to be used by tests to force no CA certs
to be loaded. Future patches will enable _defaultcacerts() to have
success more often. Without this change we can't reliably test the
failure to load CA certs. (This patch also likely fixes test failures
on some OS X configurations.)
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Fri, 01 Jul 2016 19:17:45 -0700 |
parents | 67999697221a |
children | 3dcaf1c4e90d |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15169
diff
changeset
|
9 '''base class for store implementations and store-related utility code''' |
29307
67999697221a
py3: make largefiles/basestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29305
diff
changeset
|
10 from __future__ import absolute_import |
15168 | 11 |
12 from mercurial.i18n import _ | |
13 | |
29307
67999697221a
py3: make largefiles/basestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29305
diff
changeset
|
14 from mercurial import node, util |
67999697221a
py3: make largefiles/basestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29305
diff
changeset
|
15 |
67999697221a
py3: make largefiles/basestore.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents:
29305
diff
changeset
|
16 from . import lfutil |
15168 | 17 |
18 class StoreError(Exception): | |
19 '''Raised when there is a problem getting files from or putting | |
20 files to a central store.''' | |
21 def __init__(self, filename, hash, url, detail): | |
22 self.filename = filename | |
23 self.hash = hash | |
24 self.url = url | |
25 self.detail = detail | |
26 | |
27 def longmessage(self): | |
18461
abfbb04fab8e
largefiles: enhance error message to make it more i18n-friendly
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
18155
diff
changeset
|
28 return (_("error getting id %s from url %s for file %s: %s\n") % |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
29 (self.hash, util.hidepassword(self.url), self.filename, |
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
30 self.detail)) |
15168 | 31 |
32 def __str__(self): | |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
33 return "%s: %s" % (util.hidepassword(self.url), self.detail) |
15168 | 34 |
35 class basestore(object): | |
36 def __init__(self, ui, repo, url): | |
37 self.ui = ui | |
38 self.repo = repo | |
39 self.url = url | |
40 | |
41 def put(self, source, hash): | |
19007
266b5fb72f26
largefiles: 'put' should store 'source' file in under 'hash', also in localstore
Mads Kiilerich <madski@unity3d.com>
parents:
19003
diff
changeset
|
42 '''Put source file into the store so it can be retrieved by hash.''' |
15168 | 43 raise NotImplementedError('abstract method') |
44 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
45 def exists(self, hashes): |
18573
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18546
diff
changeset
|
46 '''Check to see if the store contains the given hashes. Given an |
003730ca254d
largefiles: fold oddly named _verify into remotestore.exists
Mads Kiilerich <mads@kiilerich.com>
parents:
18546
diff
changeset
|
47 iterable of hashes it returns a mapping from hash to bool.''' |
15168 | 48 raise NotImplementedError('abstract method') |
49 | |
50 def get(self, files): | |
51 '''Get the specified largefiles from the store and write to local | |
52 files under repo.root. files is a list of (filename, hash) | |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17127
diff
changeset
|
53 tuples. Return (success, missing), lists of files successfully |
15168 | 54 downloaded and those not found in the store. success is a list |
55 of (filename, hash) tuples; missing is a list of filenames that | |
56 we could not get. (The detailed error message will already have | |
57 been presented to the user, so missing is just supplied as a | |
58 summary.)''' | |
59 success = [] | |
60 missing = [] | |
61 ui = self.ui | |
62 | |
63 at = 0 | |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
64 available = self.exists(set(hash for (_filename, hash) in files)) |
15168 | 65 for filename, hash in files: |
28463
19b4a2087dfc
largefiles: specify unit for ui.progress when operating on files
Anton Shestakov <av6@dwimlabs.net>
parents:
26587
diff
changeset
|
66 ui.progress(_('getting largefiles'), at, unit=_('files'), |
15168 | 67 total=len(files)) |
68 at += 1 | |
69 ui.note(_('getting %s:%s\n') % (filename, hash)) | |
70 | |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
71 if not available.get(hash): |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
72 ui.warn(_('%s: largefile %s not available from %s\n') |
19950
cce7ab960312
largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents:
19918
diff
changeset
|
73 % (filename, hash, util.hidepassword(self.url))) |
19008
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
74 missing.append(filename) |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
75 continue |
9d33d6e0d442
largefiles: stat all largefiles in one batch before downloading
Mads Kiilerich <madski@unity3d.com>
parents:
19007
diff
changeset
|
76 |
19918
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
77 if self._gethash(filename, hash): |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
78 success.append((filename, hash)) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
79 else: |
15168 | 80 missing.append(filename) |
81 | |
82 ui.progress(_('getting largefiles'), None) | |
83 return (success, missing) | |
84 | |
19918
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
85 def _gethash(self, filename, hash): |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
86 """Get file with the provided hash and store it in the local repo's |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
87 store and in the usercache. |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
88 filename is for informational messages only. |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
89 """ |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
90 util.makedirs(lfutil.storepath(self.repo, '')) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
91 storefilename = lfutil.storepath(self.repo, hash) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
92 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
93 tmpname = storefilename + '.tmp' |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
94 tmpfile = util.atomictempfile(tmpname, |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
95 createmode=self.repo.store.createmode) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
96 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
97 try: |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
98 gothash = self._getfile(tmpfile, filename, hash) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
23941
diff
changeset
|
99 except StoreError as err: |
19918
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
100 self.ui.warn(err.longmessage()) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
101 gothash = "" |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
102 tmpfile.close() |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
103 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
104 if gothash != hash: |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
105 if gothash != "": |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
106 self.ui.warn(_('%s: data corruption (expected %s, got %s)\n') |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
107 % (filename, hash, gothash)) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
108 util.unlink(tmpname) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
109 return False |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
110 |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
111 util.rename(tmpname, storefilename) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
112 lfutil.linktousercache(self.repo, hash) |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
113 return True |
ae65192fd6b4
largefiles: refactor basestore, extract _gethash method
Mads Kiilerich <madski@unity3d.com>
parents:
19008
diff
changeset
|
114 |
15168 | 115 def verify(self, revs, contents=False): |
116 '''Verify the existence (and, optionally, contents) of every big | |
117 file revision referenced by every changeset in revs. | |
118 Return 0 if all is well, non-zero on any errors.''' | |
119 | |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
120 self.ui.status(_('searching %d changesets for largefiles\n') % |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
121 len(revs)) |
15168 | 122 verified = set() # set of (filename, filenode) tuples |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
123 filestocheck = [] # list of (cset, filename, expectedhash) |
15168 | 124 for rev in revs: |
125 cctx = self.repo[rev] | |
126 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | |
127 | |
18486
1067a6240f86
largefiles: verify all files in each revision and report errors in any revision
Mads Kiilerich <madski@unity3d.com>
parents:
18483
diff
changeset
|
128 for standin in cctx: |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
129 filename = lfutil.splitstandin(standin) |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
130 if filename: |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
131 fctx = cctx[standin] |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
132 key = (filename, fctx.filenode()) |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
133 if key not in verified: |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
134 verified.add(key) |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
135 expectedhash = fctx.data()[0:40] |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
136 filestocheck.append((cset, filename, expectedhash)) |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
137 |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
138 failed = self._verifyfiles(contents, filestocheck) |
15168 | 139 |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
140 numrevs = len(verified) |
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16154
diff
changeset
|
141 numlfiles = len(set([fname for (fname, fnode) in verified])) |
15168 | 142 if contents: |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
143 self.ui.status( |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
144 _('verified contents of %d revisions of %d largefiles\n') |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
145 % (numrevs, numlfiles)) |
15168 | 146 else: |
18546
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
147 self.ui.status( |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
148 _('verified existence of %d revisions of %d largefiles\n') |
fb0e8966a4be
largefiles: verify status should be written as status, not as write
Mads Kiilerich <madski@unity3d.com>
parents:
18489
diff
changeset
|
149 % (numrevs, numlfiles)) |
15168 | 150 return int(failed) |
151 | |
152 def _getfile(self, tmpfile, filename, hash): | |
153 '''Fetch one revision of one file from the store and write it | |
154 to tmpfile. Compute the hash of the file on-the-fly as it | |
18999
c1b5f9c4d989
largefiles: refactoring - return hex from _getfile and copyandhash
Mads Kiilerich <madski@unity3d.com>
parents:
18731
diff
changeset
|
155 downloads and return the hash. Close tmpfile. Raise |
15168 | 156 StoreError if unable to download the file (e.g. it does not |
157 exist in the store).''' | |
158 raise NotImplementedError('abstract method') | |
159 | |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
160 def _verifyfiles(self, contents, filestocheck): |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
161 '''Perform the actual verification of files in the store. |
18574
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18573
diff
changeset
|
162 'contents' controls verification of content hash. |
29067
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
163 'filestocheck' is list of files to check. |
207c0db08953
largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents:
28463
diff
changeset
|
164 Returns _true_ if any problems are found! |
15168 | 165 ''' |
166 raise NotImplementedError('abstract method') |