Mercurial > hg
comparison hgext/largefiles/basestore.py @ 15168:cfccd3bee7b3
hgext: add largefiles extension
This code has a number of contributors and a complicated history prior to its
introduction that can be seen by visiting:
https://developers.kilnhg.com/Repo/Kiln/largefiles/largefiles
http://hg.gerg.ca/hg-bfiles
and looking at the included copyright notices and contributors list.
author | various |
---|---|
date | Sat, 24 Sep 2011 17:35:45 +0200 |
parents | |
children | aa262fff87ac |
comparison
equal
deleted
inserted
replaced
15167:8df4166b6f63 | 15168:cfccd3bee7b3 |
---|---|
1 # Copyright 2009-2010 Gregory P. Ward | |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''Base class for store implementations and store-related utility code.''' | |
10 | |
11 import os | |
12 import tempfile | |
13 import binascii | |
14 import re | |
15 | |
16 from mercurial import util, node, hg | |
17 from mercurial.i18n import _ | |
18 | |
19 import lfutil | |
20 | |
21 class StoreError(Exception): | |
22 '''Raised when there is a problem getting files from or putting | |
23 files to a central store.''' | |
24 def __init__(self, filename, hash, url, detail): | |
25 self.filename = filename | |
26 self.hash = hash | |
27 self.url = url | |
28 self.detail = detail | |
29 | |
30 def longmessage(self): | |
31 if self.url: | |
32 return ('%s: %s\n' | |
33 '(failed URL: %s)\n' | |
34 % (self.filename, self.detail, self.url)) | |
35 else: | |
36 return ('%s: %s\n' | |
37 '(no default or default-push path set in hgrc)\n' | |
38 % (self.filename, self.detail)) | |
39 | |
40 def __str__(self): | |
41 return "%s: %s" % (self.url, self.detail) | |
42 | |
43 class basestore(object): | |
44 def __init__(self, ui, repo, url): | |
45 self.ui = ui | |
46 self.repo = repo | |
47 self.url = url | |
48 | |
49 def put(self, source, hash): | |
50 '''Put source file into the store under <filename>/<hash>.''' | |
51 raise NotImplementedError('abstract method') | |
52 | |
53 def exists(self, hash): | |
54 '''Check to see if the store contains the given hash.''' | |
55 raise NotImplementedError('abstract method') | |
56 | |
57 def get(self, files): | |
58 '''Get the specified largefiles from the store and write to local | |
59 files under repo.root. files is a list of (filename, hash) | |
60 tuples. Return (success, missing), lists of files successfuly | |
61 downloaded and those not found in the store. success is a list | |
62 of (filename, hash) tuples; missing is a list of filenames that | |
63 we could not get. (The detailed error message will already have | |
64 been presented to the user, so missing is just supplied as a | |
65 summary.)''' | |
66 success = [] | |
67 missing = [] | |
68 ui = self.ui | |
69 | |
70 at = 0 | |
71 for filename, hash in files: | |
72 ui.progress(_('getting largefiles'), at, unit='lfile', | |
73 total=len(files)) | |
74 at += 1 | |
75 ui.note(_('getting %s:%s\n') % (filename, hash)) | |
76 | |
77 cachefilename = lfutil.cachepath(self.repo, hash) | |
78 cachedir = os.path.dirname(cachefilename) | |
79 | |
80 # No need to pass mode='wb' to fdopen(), since mkstemp() already | |
81 # opened the file in binary mode. | |
82 (tmpfd, tmpfilename) = tempfile.mkstemp( | |
83 dir=cachedir, prefix=os.path.basename(filename)) | |
84 tmpfile = os.fdopen(tmpfd, 'w') | |
85 | |
86 try: | |
87 hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash)) | |
88 except StoreError, err: | |
89 ui.warn(err.longmessage()) | |
90 hhash = "" | |
91 | |
92 if hhash != hash: | |
93 if hhash != "": | |
94 ui.warn(_('%s: data corruption (expected %s, got %s)\n') | |
95 % (filename, hash, hhash)) | |
96 tmpfile.close() # no-op if it's already closed | |
97 os.remove(tmpfilename) | |
98 missing.append(filename) | |
99 continue | |
100 | |
101 if os.path.exists(cachefilename): # Windows | |
102 os.remove(cachefilename) | |
103 os.rename(tmpfilename, cachefilename) | |
104 lfutil.linktosystemcache(self.repo, hash) | |
105 success.append((filename, hhash)) | |
106 | |
107 ui.progress(_('getting largefiles'), None) | |
108 return (success, missing) | |
109 | |
110 def verify(self, revs, contents=False): | |
111 '''Verify the existence (and, optionally, contents) of every big | |
112 file revision referenced by every changeset in revs. | |
113 Return 0 if all is well, non-zero on any errors.''' | |
114 write = self.ui.write | |
115 failed = False | |
116 | |
117 write(_('searching %d changesets for largefiles\n') % len(revs)) | |
118 verified = set() # set of (filename, filenode) tuples | |
119 | |
120 for rev in revs: | |
121 cctx = self.repo[rev] | |
122 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | |
123 | |
124 failed = lfutil.any_(self._verifyfile( | |
125 cctx, cset, contents, standin, verified) for standin in cctx) | |
126 | |
127 num_revs = len(verified) | |
128 num_lfiles = len(set([fname for (fname, fnode) in verified])) | |
129 if contents: | |
130 write(_('verified contents of %d revisions of %d largefiles\n') | |
131 % (num_revs, num_lfiles)) | |
132 else: | |
133 write(_('verified existence of %d revisions of %d largefiles\n') | |
134 % (num_revs, num_lfiles)) | |
135 | |
136 return int(failed) | |
137 | |
138 def _getfile(self, tmpfile, filename, hash): | |
139 '''Fetch one revision of one file from the store and write it | |
140 to tmpfile. Compute the hash of the file on-the-fly as it | |
141 downloads and return the binary hash. Close tmpfile. Raise | |
142 StoreError if unable to download the file (e.g. it does not | |
143 exist in the store).''' | |
144 raise NotImplementedError('abstract method') | |
145 | |
146 def _verifyfile(self, cctx, cset, contents, standin, verified): | |
147 '''Perform the actual verification of a file in the store. | |
148 ''' | |
149 raise NotImplementedError('abstract method') | |
150 | |
151 import localstore, wirestore | |
152 | |
153 _storeprovider = { | |
154 'file': [localstore.localstore], | |
155 'http': [wirestore.wirestore], | |
156 'https': [wirestore.wirestore], | |
157 'ssh': [wirestore.wirestore], | |
158 } | |
159 | |
160 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') | |
161 | |
162 # During clone this function is passed the src's ui object | |
163 # but it needs the dest's ui object so it can read out of | |
164 # the config file. Use repo.ui instead. | |
165 def _openstore(repo, remote=None, put=False): | |
166 ui = repo.ui | |
167 | |
168 if not remote: | |
169 path = getattr(repo, 'lfpullsource', None) or \ | |
170 ui.expandpath('default-push', 'default') | |
171 # If 'default-push' and 'default' can't be expanded | |
172 # they are just returned. In that case use the empty string which | |
173 # use the filescheme. | |
174 if path == 'default-push' or path == 'default': | |
175 path = '' | |
176 remote = repo | |
177 else: | |
178 remote = hg.peer(repo, {}, path) | |
179 | |
180 # The path could be a scheme so use Mercurial's normal functionality | |
181 # to resolve the scheme to a repository and use its path | |
182 path = hasattr(remote, 'url') and remote.url() or remote.path | |
183 | |
184 match = _scheme_re.match(path) | |
185 if not match: # regular filesystem path | |
186 scheme = 'file' | |
187 else: | |
188 scheme = match.group(1) | |
189 | |
190 try: | |
191 storeproviders = _storeprovider[scheme] | |
192 except KeyError: | |
193 raise util.Abort(_('unsupported URL scheme %r') % scheme) | |
194 | |
195 for class_obj in storeproviders: | |
196 try: | |
197 return class_obj(ui, repo, remote) | |
198 except lfutil.storeprotonotcapable: | |
199 pass | |
200 | |
201 raise util.Abort(_('%s does not appear to be a lfile store'), path) |