author | Na'Tosha Bard <natosha@unity3d.com> |
Thu, 06 Oct 2011 11:10:06 +0200 | |
changeset 15206 | f85c76b16f27 |
parent 15188 | 8e115063950d |
child 15224 | 7c604d8c7e83 |
permissions | -rw-r--r-- |
15168 | 1 |
# Copyright 2009-2010 Gregory P. Ward |
2 |
# Copyright 2009-2010 Intelerad Medical Systems Incorporated |
|
3 |
# Copyright 2010-2011 Fog Creek Software |
|
4 |
# Copyright 2010-2011 Unity Technologies |
|
5 |
# |
|
6 |
# This software may be used and distributed according to the terms of the |
|
7 |
# GNU General Public License version 2 or any later version. |
|
8 |
||
9 |
'''largefiles utility code: must not import other modules in this package.''' |
|
10 |
||
11 |
import os |
|
12 |
import errno |
|
13 |
import inspect |
|
14 |
import shutil |
|
15 |
import stat |
|
16 |
import hashlib |
|
17 |
||
18 |
from mercurial import cmdutil, dirstate, httpconnection, match as match_, \ |
|
19 |
url as url_, util |
|
20 |
from mercurial.i18n import _ |
|
21 |
||
22 |
try: |
|
23 |
from mercurial import scmutil |
|
24 |
except ImportError: |
|
25 |
pass |
|
26 |
||
27 |
shortname = '.hglf' |
|
28 |
longname = 'largefiles' |
|
29 |
||
30 |
||
31 |
# -- Portability wrappers ---------------------------------------------- |
|
32 |
||
33 |
if 'subrepos' in inspect.getargspec(dirstate.dirstate.status)[0]: |
|
34 |
# for Mercurial >= 1.5 |
|
35 |
def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
|
36 |
return dirstate.walk(matcher, [], unknown, ignored) |
|
37 |
else: |
|
38 |
# for Mercurial <= 1.4 |
|
39 |
def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
|
40 |
return dirstate.walk(matcher, unknown, ignored) |
|
41 |
||
42 |
def repo_add(repo, list): |
|
43 |
try: |
|
44 |
# Mercurial <= 1.5 |
|
45 |
add = repo.add |
|
46 |
except AttributeError: |
|
47 |
# Mercurial >= 1.6 |
|
48 |
add = repo[None].add |
|
49 |
return add(list) |
|
50 |
||
51 |
def repo_remove(repo, list, unlink=False): |
|
52 |
try: |
|
53 |
# Mercurial <= 1.5 |
|
54 |
remove = repo.remove |
|
55 |
except AttributeError: |
|
56 |
# Mercurial >= 1.6 |
|
57 |
try: |
|
58 |
# Mercurial <= 1.8 |
|
59 |
remove = repo[None].remove |
|
60 |
except AttributeError: |
|
61 |
# Mercurial >= 1.9 |
|
62 |
def remove(list, unlink): |
|
63 |
wlock = repo.wlock() |
|
64 |
try: |
|
65 |
if unlink: |
|
66 |
for f in list: |
|
67 |
try: |
|
68 |
util.unlinkpath(repo.wjoin(f)) |
|
69 |
except OSError, inst: |
|
70 |
if inst.errno != errno.ENOENT: |
|
71 |
raise |
|
72 |
repo[None].forget(list) |
|
73 |
finally: |
|
74 |
wlock.release() |
|
75 |
||
76 |
return remove(list, unlink=unlink) |
|
77 |
||
78 |
def repo_forget(repo, list): |
|
79 |
try: |
|
80 |
# Mercurial <= 1.5 |
|
81 |
forget = repo.forget |
|
82 |
except AttributeError: |
|
83 |
# Mercurial >= 1.6 |
|
84 |
forget = repo[None].forget |
|
85 |
return forget(list) |
|
86 |
||
87 |
def findoutgoing(repo, remote, force): |
|
88 |
# First attempt is for Mercurial <= 1.5 second is for >= 1.6 |
|
89 |
try: |
|
90 |
return repo.findoutgoing(remote) |
|
91 |
except AttributeError: |
|
92 |
from mercurial import discovery |
|
93 |
try: |
|
94 |
# Mercurial <= 1.8 |
|
95 |
return discovery.findoutgoing(repo, remote, force=force) |
|
96 |
except AttributeError: |
|
97 |
# Mercurial >= 1.9 |
|
98 |
common, _anyinc, _heads = discovery.findcommonincoming(repo, |
|
99 |
remote, force=force) |
|
100 |
return repo.changelog.findmissing(common) |
|
101 |
||
102 |
# -- Private worker functions ------------------------------------------ |
|
103 |
||
104 |
def link(src, dest): |
|
105 |
try: |
|
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
106 |
util.oslink(src, dest) |
15168 | 107 |
except OSError: |
108 |
# If hardlinks fail fall back on copy |
|
109 |
shutil.copyfile(src, dest) |
|
110 |
os.chmod(dest, os.stat(src).st_mode) |
|
111 |
||
112 |
def systemcachepath(ui, hash): |
|
113 |
path = ui.config(longname, 'systemcache', None) |
|
114 |
if path: |
|
115 |
path = os.path.join(path, hash) |
|
116 |
else: |
|
117 |
if os.name == 'nt': |
|
118 |
path = os.path.join(os.getenv('LOCALAPPDATA') or \ |
|
119 |
os.getenv('APPDATA'), longname, hash) |
|
120 |
elif os.name == 'posix': |
|
121 |
path = os.path.join(os.getenv('HOME'), '.' + longname, hash) |
|
122 |
else: |
|
123 |
raise util.Abort(_('Unknown operating system: %s\n') % os.name) |
|
124 |
return path |
|
125 |
||
126 |
def insystemcache(ui, hash): |
|
127 |
return os.path.exists(systemcachepath(ui, hash)) |
|
128 |
||
129 |
def findfile(repo, hash): |
|
130 |
if incache(repo, hash): |
|
131 |
repo.ui.note(_('Found %s in cache\n') % hash) |
|
132 |
return cachepath(repo, hash) |
|
133 |
if insystemcache(repo.ui, hash): |
|
134 |
repo.ui.note(_('Found %s in system cache\n') % hash) |
|
135 |
return systemcachepath(repo.ui, hash) |
|
136 |
return None |
|
137 |
||
138 |
class largefiles_dirstate(dirstate.dirstate): |
|
139 |
def __getitem__(self, key): |
|
140 |
return super(largefiles_dirstate, self).__getitem__(unixpath(key)) |
|
141 |
def normal(self, f): |
|
142 |
return super(largefiles_dirstate, self).normal(unixpath(f)) |
|
143 |
def remove(self, f): |
|
144 |
return super(largefiles_dirstate, self).remove(unixpath(f)) |
|
145 |
def add(self, f): |
|
146 |
return super(largefiles_dirstate, self).add(unixpath(f)) |
|
147 |
def drop(self, f): |
|
148 |
return super(largefiles_dirstate, self).drop(unixpath(f)) |
|
149 |
def forget(self, f): |
|
150 |
return super(largefiles_dirstate, self).forget(unixpath(f)) |
|
151 |
||
152 |
def openlfdirstate(ui, repo): |
|
153 |
''' |
|
154 |
Return a dirstate object that tracks big files: i.e. its root is the |
|
155 |
repo root, but it is saved in .hg/largefiles/dirstate. |
|
156 |
''' |
|
157 |
admin = repo.join(longname) |
|
158 |
try: |
|
159 |
# Mercurial >= 1.9 |
|
160 |
opener = scmutil.opener(admin) |
|
161 |
except ImportError: |
|
162 |
# Mercurial <= 1.8 |
|
163 |
opener = util.opener(admin) |
|
15169
aa262fff87ac
largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
164 |
if util.safehasattr(repo.dirstate, '_validate'): |
15168 | 165 |
lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
166 |
repo.dirstate._validate) |
|
167 |
else: |
|
168 |
lfdirstate = largefiles_dirstate(opener, ui, repo.root) |
|
169 |
||
170 |
# If the largefiles dirstate does not exist, populate and create it. This |
|
171 |
# ensures that we create it on the first meaningful largefiles operation in |
|
172 |
# a new clone. It also gives us an easy way to forcibly rebuild largefiles |
|
173 |
# state: |
|
174 |
# rm .hg/largefiles/dirstate && hg status |
|
175 |
# Or even, if things are really messed up: |
|
176 |
# rm -rf .hg/largefiles && hg status |
|
177 |
if not os.path.exists(os.path.join(admin, 'dirstate')): |
|
178 |
util.makedirs(admin) |
|
179 |
matcher = getstandinmatcher(repo) |
|
180 |
for standin in dirstate_walk(repo.dirstate, matcher): |
|
181 |
lfile = splitstandin(standin) |
|
182 |
hash = readstandin(repo, lfile) |
|
183 |
lfdirstate.normallookup(lfile) |
|
184 |
try: |
|
185 |
if hash == hashfile(lfile): |
|
186 |
lfdirstate.normal(lfile) |
|
187 |
except IOError, err: |
|
188 |
if err.errno != errno.ENOENT: |
|
189 |
raise |
|
190 |
||
191 |
lfdirstate.write() |
|
192 |
||
193 |
return lfdirstate |
|
194 |
||
195 |
def lfdirstate_status(lfdirstate, repo, rev): |
|
196 |
wlock = repo.wlock() |
|
197 |
try: |
|
198 |
match = match_.always(repo.root, repo.getcwd()) |
|
199 |
s = lfdirstate.status(match, [], False, False, False) |
|
200 |
unsure, modified, added, removed, missing, unknown, ignored, clean = s |
|
201 |
for lfile in unsure: |
|
202 |
if repo[rev][standin(lfile)].data().strip() != \ |
|
203 |
hashfile(repo.wjoin(lfile)): |
|
204 |
modified.append(lfile) |
|
205 |
else: |
|
206 |
clean.append(lfile) |
|
207 |
lfdirstate.normal(lfile) |
|
208 |
lfdirstate.write() |
|
209 |
finally: |
|
210 |
wlock.release() |
|
211 |
return (modified, added, removed, missing, unknown, ignored, clean) |
|
212 |
||
213 |
def listlfiles(repo, rev=None, matcher=None): |
|
214 |
'''list largefiles in the working copy or specified changeset''' |
|
215 |
||
216 |
if matcher is None: |
|
217 |
matcher = getstandinmatcher(repo) |
|
218 |
||
219 |
# ignore unknown files in working directory |
|
220 |
return [splitstandin(f) for f in repo[rev].walk(matcher) \ |
|
221 |
if rev is not None or repo.dirstate[f] != '?'] |
|
222 |
||
223 |
def incache(repo, hash): |
|
224 |
return os.path.exists(cachepath(repo, hash)) |
|
225 |
||
226 |
def createdir(dir): |
|
227 |
if not os.path.exists(dir): |
|
228 |
os.makedirs(dir) |
|
229 |
||
230 |
def cachepath(repo, hash): |
|
231 |
return repo.join(os.path.join(longname, hash)) |
|
232 |
||
233 |
def copyfromcache(repo, hash, filename): |
|
234 |
'''copyfromcache copies the specified largefile from the repo or system |
|
235 |
cache to the specified location in the repository. It will not throw an |
|
236 |
exception on failure, as it is meant to be called only after ensuring that |
|
237 |
the needed largefile exists in the cache.''' |
|
238 |
path = findfile(repo, hash) |
|
239 |
if path is None: |
|
240 |
return False |
|
241 |
util.makedirs(os.path.dirname(repo.wjoin(filename))) |
|
242 |
shutil.copy(path, repo.wjoin(filename)) |
|
243 |
return True |
|
244 |
||
245 |
def copytocache(repo, rev, file, uploaded=False): |
|
246 |
hash = readstandin(repo, file) |
|
247 |
if incache(repo, hash): |
|
248 |
return |
|
249 |
copytocacheabsolute(repo, repo.wjoin(file), hash) |
|
250 |
||
251 |
def copytocacheabsolute(repo, file, hash): |
|
252 |
createdir(os.path.dirname(cachepath(repo, hash))) |
|
253 |
if insystemcache(repo.ui, hash): |
|
254 |
link(systemcachepath(repo.ui, hash), cachepath(repo, hash)) |
|
255 |
else: |
|
256 |
shutil.copyfile(file, cachepath(repo, hash)) |
|
257 |
os.chmod(cachepath(repo, hash), os.stat(file).st_mode) |
|
258 |
linktosystemcache(repo, hash) |
|
259 |
||
260 |
def linktosystemcache(repo, hash): |
|
261 |
createdir(os.path.dirname(systemcachepath(repo.ui, hash))) |
|
262 |
link(cachepath(repo, hash), systemcachepath(repo.ui, hash)) |
|
263 |
||
264 |
def getstandinmatcher(repo, pats=[], opts={}): |
|
265 |
'''Return a match object that applies pats to the standin directory''' |
|
266 |
standindir = repo.pathto(shortname) |
|
267 |
if pats: |
|
268 |
# patterns supplied: search standin directory relative to current dir |
|
269 |
cwd = repo.getcwd() |
|
270 |
if os.path.isabs(cwd): |
|
271 |
# cwd is an absolute path for hg -R <reponame> |
|
272 |
# work relative to the repository root in this case |
|
273 |
cwd = '' |
|
274 |
pats = [os.path.join(standindir, cwd, pat) for pat in pats] |
|
275 |
elif os.path.isdir(standindir): |
|
276 |
# no patterns: relative to repo root |
|
277 |
pats = [standindir] |
|
278 |
else: |
|
279 |
# no patterns and no standin dir: return matcher that matches nothing |
|
280 |
match = match_.match(repo.root, None, [], exact=True) |
|
281 |
match.matchfn = lambda f: False |
|
282 |
return match |
|
283 |
return getmatcher(repo, pats, opts, showbad=False) |
|
284 |
||
285 |
def getmatcher(repo, pats=[], opts={}, showbad=True): |
|
286 |
'''Wrapper around scmutil.match() that adds showbad: if false, neuter |
|
287 |
the match object\'s bad() method so it does not print any warnings |
|
288 |
about missing files or directories.''' |
|
289 |
try: |
|
290 |
# Mercurial >= 1.9 |
|
291 |
match = scmutil.match(repo[None], pats, opts) |
|
292 |
except ImportError: |
|
293 |
# Mercurial <= 1.8 |
|
294 |
match = cmdutil.match(repo, pats, opts) |
|
295 |
||
296 |
if not showbad: |
|
297 |
match.bad = lambda f, msg: None |
|
298 |
return match |
|
299 |
||
300 |
def composestandinmatcher(repo, rmatcher): |
|
301 |
'''Return a matcher that accepts standins corresponding to the files |
|
302 |
accepted by rmatcher. Pass the list of files in the matcher as the |
|
303 |
paths specified by the user.''' |
|
304 |
smatcher = getstandinmatcher(repo, rmatcher.files()) |
|
305 |
isstandin = smatcher.matchfn |
|
306 |
def composed_matchfn(f): |
|
307 |
return isstandin(f) and rmatcher.matchfn(splitstandin(f)) |
|
308 |
smatcher.matchfn = composed_matchfn |
|
309 |
||
310 |
return smatcher |
|
311 |
||
312 |
def standin(filename): |
|
313 |
'''Return the repo-relative path to the standin for the specified big |
|
314 |
file.''' |
|
315 |
# Notes: |
|
316 |
# 1) Most callers want an absolute path, but _create_standin() needs |
|
317 |
# it repo-relative so lfadd() can pass it to repo_add(). So leave |
|
318 |
# it up to the caller to use repo.wjoin() to get an absolute path. |
|
319 |
# 2) Join with '/' because that's what dirstate always uses, even on |
|
320 |
# Windows. Change existing separator to '/' first in case we are |
|
321 |
# passed filenames from an external source (like the command line). |
|
322 |
return shortname + '/' + filename.replace(os.sep, '/') |
|
323 |
||
324 |
def isstandin(filename): |
|
325 |
'''Return true if filename is a big file standin. filename must |
|
326 |
be in Mercurial\'s internal form (slash-separated).''' |
|
327 |
return filename.startswith(shortname + '/') |
|
328 |
||
329 |
def splitstandin(filename): |
|
330 |
# Split on / because that's what dirstate always uses, even on Windows. |
|
331 |
# Change local separator to / first just in case we are passed filenames |
|
332 |
# from an external source (like the command line). |
|
333 |
bits = filename.replace(os.sep, '/').split('/', 1) |
|
334 |
if len(bits) == 2 and bits[0] == shortname: |
|
335 |
return bits[1] |
|
336 |
else: |
|
337 |
return None |
|
338 |
||
339 |
def updatestandin(repo, standin): |
|
340 |
file = repo.wjoin(splitstandin(standin)) |
|
341 |
if os.path.exists(file): |
|
342 |
hash = hashfile(file) |
|
343 |
executable = getexecutable(file) |
|
344 |
writestandin(repo, standin, hash, executable) |
|
345 |
||
346 |
def readstandin(repo, filename, node=None): |
|
347 |
'''read hex hash from standin for filename at given node, or working |
|
348 |
directory if no node is given''' |
|
349 |
return repo[node][standin(filename)].data().strip() |
|
350 |
||
351 |
def writestandin(repo, standin, hash, executable): |
|
352 |
'''write hhash to <repo.root>/<standin>''' |
|
353 |
writehash(hash, repo.wjoin(standin), executable) |
|
354 |
||
355 |
def copyandhash(instream, outfile): |
|
356 |
'''Read bytes from instream (iterable) and write them to outfile, |
|
357 |
computing the SHA-1 hash of the data along the way. Close outfile |
|
358 |
when done and return the binary hash.''' |
|
359 |
hasher = util.sha1('') |
|
360 |
for data in instream: |
|
361 |
hasher.update(data) |
|
362 |
outfile.write(data) |
|
363 |
||
364 |
# Blecch: closing a file that somebody else opened is rude and |
|
365 |
# wrong. But it's so darn convenient and practical! After all, |
|
366 |
# outfile was opened just to copy and hash. |
|
367 |
outfile.close() |
|
368 |
||
369 |
return hasher.digest() |
|
370 |
||
371 |
def hashrepofile(repo, file): |
|
372 |
return hashfile(repo.wjoin(file)) |
|
373 |
||
374 |
def hashfile(file): |
|
375 |
if not os.path.exists(file): |
|
376 |
return '' |
|
377 |
hasher = util.sha1('') |
|
378 |
fd = open(file, 'rb') |
|
379 |
for data in blockstream(fd): |
|
380 |
hasher.update(data) |
|
381 |
fd.close() |
|
382 |
return hasher.hexdigest() |
|
383 |
||
384 |
class limitreader(object): |
|
385 |
def __init__(self, f, limit): |
|
386 |
self.f = f |
|
387 |
self.limit = limit |
|
388 |
||
389 |
def read(self, length): |
|
390 |
if self.limit == 0: |
|
391 |
return '' |
|
392 |
length = length > self.limit and self.limit or length |
|
393 |
self.limit -= length |
|
394 |
return self.f.read(length) |
|
395 |
||
396 |
def close(self): |
|
397 |
pass |
|
398 |
||
399 |
def blockstream(infile, blocksize=128 * 1024): |
|
400 |
"""Generator that yields blocks of data from infile and closes infile.""" |
|
401 |
while True: |
|
402 |
data = infile.read(blocksize) |
|
403 |
if not data: |
|
404 |
break |
|
405 |
yield data |
|
406 |
# Same blecch as above. |
|
407 |
infile.close() |
|
408 |
||
409 |
def readhash(filename): |
|
410 |
rfile = open(filename, 'rb') |
|
411 |
hash = rfile.read(40) |
|
412 |
rfile.close() |
|
413 |
if len(hash) < 40: |
|
414 |
raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') |
|
415 |
% (filename, len(hash))) |
|
416 |
return hash |
|
417 |
||
418 |
def writehash(hash, filename, executable): |
|
419 |
util.makedirs(os.path.dirname(filename)) |
|
420 |
if os.path.exists(filename): |
|
421 |
os.unlink(filename) |
|
422 |
wfile = open(filename, 'wb') |
|
423 |
||
424 |
try: |
|
425 |
wfile.write(hash) |
|
426 |
wfile.write('\n') |
|
427 |
finally: |
|
428 |
wfile.close() |
|
429 |
if os.path.exists(filename): |
|
430 |
os.chmod(filename, getmode(executable)) |
|
431 |
||
432 |
def getexecutable(filename): |
|
433 |
mode = os.stat(filename).st_mode |
|
434 |
return (mode & stat.S_IXUSR) and (mode & stat.S_IXGRP) and (mode & \ |
|
435 |
stat.S_IXOTH) |
|
436 |
||
437 |
def getmode(executable): |
|
438 |
if executable: |
|
439 |
return 0755 |
|
440 |
else: |
|
441 |
return 0644 |
|
442 |
||
443 |
def urljoin(first, second, *arg): |
|
444 |
def join(left, right): |
|
445 |
if not left.endswith('/'): |
|
446 |
left += '/' |
|
447 |
if right.startswith('/'): |
|
448 |
right = right[1:] |
|
449 |
return left + right |
|
450 |
||
451 |
url = join(first, second) |
|
452 |
for a in arg: |
|
453 |
url = join(url, a) |
|
454 |
return url |
|
455 |
||
456 |
def hexsha1(data): |
|
457 |
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like |
|
458 |
object data""" |
|
459 |
h = hashlib.sha1() |
|
460 |
for chunk in util.filechunkiter(data): |
|
461 |
h.update(chunk) |
|
462 |
return h.hexdigest() |
|
463 |
||
464 |
def httpsendfile(ui, filename): |
|
465 |
try: |
|
466 |
# Mercurial >= 1.9 |
|
467 |
return httpconnection.httpsendfile(ui, filename, 'rb') |
|
468 |
except ImportError: |
|
469 |
if 'ui' in inspect.getargspec(url_.httpsendfile.__init__)[0]: |
|
470 |
# Mercurial == 1.8 |
|
471 |
return url_.httpsendfile(ui, filename, 'rb') |
|
472 |
else: |
|
473 |
# Mercurial <= 1.7 |
|
474 |
return url_.httpsendfile(filename, 'rb') |
|
475 |
||
476 |
# Convert a path to a unix style path. This is used to give a |
|
477 |
# canonical path to the lfdirstate. |
|
478 |
def unixpath(path): |
|
479 |
return os.path.normpath(path).replace(os.sep, '/') |
|
480 |
||
481 |
def islfilesrepo(repo): |
|
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
482 |
return ('largefiles' in repo.requirements and |
15188
8e115063950d
largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents:
15171
diff
changeset
|
483 |
any_(shortname + '/' in f[0] for f in repo.store.datafiles())) |
15168 | 484 |
|
485 |
def any_(gen): |
|
486 |
for x in gen: |
|
487 |
if x: |
|
488 |
return True |
|
489 |
return False |
|
490 |
||
491 |
class storeprotonotcapable(BaseException): |
|
492 |
def __init__(self, storetypes): |
|
493 |
self.storetypes = storetypes |