annotate hgext/largefiles/lfutil.py @ 15794:0d91211dd12f

largefiles: fix inappropriate locking (issue3182) Don't lock/write on operations that should be readonly (status). Always lock when writing the lfdirstate (rollback). Don't write lfdirstate until after committing; state isn't actually changed until the commit is complete.
author Levi Bard <levi@unity3d.com>
date Sat, 07 Jan 2012 19:05:59 +0100
parents 3ef07ecdb0d5
children 3e5b6045ccfc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
1 # Copyright 2009-2010 Gregory P. Ward
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
3 # Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
4 # Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
5 #
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
6 # This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
7 # GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
8
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
9 '''largefiles utility code: must not import other modules in this package.'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
10
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
11 import os
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
12 import errno
15320
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
13 import platform
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
14 import shutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
15 import stat
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
16 import tempfile
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
17
15226
2223ea21c98f largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents: 15224
diff changeset
18 from mercurial import dirstate, httpconnection, match as match_, util, scmutil
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
19 from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
20
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
21 shortname = '.hglf'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
22 longname = 'largefiles'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
23
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
24
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
25 # -- Portability wrappers ----------------------------------------------
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
26
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
27 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False):
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
28 return dirstate.walk(matcher, [], unknown, ignored)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
29
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
30 def repo_add(repo, list):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
31 add = repo[None].add
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
32 return add(list)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
33
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
34 def repo_remove(repo, list, unlink=False):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
35 def remove(list, unlink):
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
36 wlock = repo.wlock()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
37 try:
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
38 if unlink:
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
39 for f in list:
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
40 try:
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
41 util.unlinkpath(repo.wjoin(f))
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
42 except OSError, inst:
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
43 if inst.errno != errno.ENOENT:
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
44 raise
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
45 repo[None].forget(list)
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
46 finally:
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
47 wlock.release()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
48 return remove(list, unlink=unlink)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
49
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
50 def repo_forget(repo, list):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
51 forget = repo[None].forget
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
52 return forget(list)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
53
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
54 def findoutgoing(repo, remote, force):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
55 from mercurial import discovery
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
56 common, _anyinc, _heads = discovery.findcommonincoming(repo,
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
57 remote, force=force)
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
58 return repo.changelog.findmissing(common)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
59
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
60 # -- Private worker functions ------------------------------------------
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
61
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
62 def getminsize(ui, assumelfiles, opt, default=10):
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
63 lfsize = opt
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
64 if not lfsize and assumelfiles:
15304
9aa9d4bb3d88 largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents: 15255
diff changeset
65 lfsize = ui.config(longname, 'minsize', default=default)
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
66 if lfsize:
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
67 try:
15228
ee625de3541e largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents: 15227
diff changeset
68 lfsize = float(lfsize)
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
69 except ValueError:
15228
ee625de3541e largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents: 15227
diff changeset
70 raise util.Abort(_('largefiles: size must be number (not %s)\n')
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
71 % lfsize)
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
72 if lfsize is None:
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
73 raise util.Abort(_('minimum size for largefiles must be specified'))
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
74 return lfsize
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
75
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
76 def link(src, dest):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
77 try:
15206
f85c76b16f27 largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents: 15188
diff changeset
78 util.oslink(src, dest)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
79 except OSError:
15572
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
80 # if hardlinks fail, fallback on atomic copy
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
81 dst = util.atomictempfile(dest)
15699
84e55467093c largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents: 15658
diff changeset
82 for chunk in util.filechunkiter(open(src, 'rb')):
15572
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
83 dst.write(chunk)
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
84 dst.close()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
85 os.chmod(dest, os.stat(src).st_mode)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
86
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
87 def usercachepath(ui, hash):
15350
8b8dd13295db largefiles: use ui.configpath() where appropriate
Greg Ward <greg@gerg.ca>
parents: 15349
diff changeset
88 path = ui.configpath(longname, 'usercache', None)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
89 if path:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
90 path = os.path.join(path, hash)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
91 else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
92 if os.name == 'nt':
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
93 appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA'))
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
94 if appdata:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
95 path = os.path.join(appdata, longname, hash)
15320
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
96 elif platform.system() == 'Darwin':
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
97 home = os.getenv('HOME')
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
98 if home:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
99 path = os.path.join(home, 'Library', 'Caches',
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
100 longname, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
101 elif os.name == 'posix':
15320
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
102 path = os.getenv('XDG_CACHE_HOME')
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
103 if path:
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
104 path = os.path.join(path, longname, hash)
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
105 else:
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
106 home = os.getenv('HOME')
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
107 if home:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
108 path = os.path.join(home, '.cache', longname, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
109 else:
15253
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
110 raise util.Abort(_('unknown operating system: %s\n') % os.name)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
111 return path
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
112
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
113 def inusercache(ui, hash):
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
114 path = usercachepath(ui, hash)
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
115 return path and os.path.exists(path)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
116
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
117 def findfile(repo, hash):
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
118 if instore(repo, hash):
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
119 repo.ui.note(_('Found %s in store\n') % hash)
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
120 elif inusercache(repo.ui, hash):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
121 repo.ui.note(_('Found %s in system cache\n') % hash)
15408
db8b0ee74025 largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents: 15392
diff changeset
122 path = storepath(repo, hash)
db8b0ee74025 largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents: 15392
diff changeset
123 util.makedirs(os.path.dirname(path))
db8b0ee74025 largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents: 15392
diff changeset
124 link(usercachepath(repo.ui, hash), path)
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
125 else:
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
126 return None
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
127 return storepath(repo, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
128
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
129 class largefiles_dirstate(dirstate.dirstate):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
130 def __getitem__(self, key):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
131 return super(largefiles_dirstate, self).__getitem__(unixpath(key))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
132 def normal(self, f):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
133 return super(largefiles_dirstate, self).normal(unixpath(f))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
134 def remove(self, f):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
135 return super(largefiles_dirstate, self).remove(unixpath(f))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
136 def add(self, f):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
137 return super(largefiles_dirstate, self).add(unixpath(f))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
138 def drop(self, f):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
139 return super(largefiles_dirstate, self).drop(unixpath(f))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
140 def forget(self, f):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
141 return super(largefiles_dirstate, self).forget(unixpath(f))
15793
3ef07ecdb0d5 largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents: 15700
diff changeset
142 def normallookup(self, f):
3ef07ecdb0d5 largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents: 15700
diff changeset
143 return super(largefiles_dirstate, self).normallookup(unixpath(f))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
144
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
145 def openlfdirstate(ui, repo):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
146 '''
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
147 Return a dirstate object that tracks largefiles: i.e. its root is
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
148 the repo root, but it is saved in .hg/largefiles/dirstate.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
149 '''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
150 admin = repo.join(longname)
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
151 opener = scmutil.opener(admin)
15349
63455eb771af largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents: 15347
diff changeset
152 lfdirstate = largefiles_dirstate(opener, ui, repo.root,
63455eb771af largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents: 15347
diff changeset
153 repo.dirstate._validate)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
154
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
155 # If the largefiles dirstate does not exist, populate and create
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
156 # it. This ensures that we create it on the first meaningful
15794
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
157 # largefiles operation in a new clone.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
158 if not os.path.exists(os.path.join(admin, 'dirstate')):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
159 util.makedirs(admin)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
160 matcher = getstandinmatcher(repo)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
161 for standin in dirstate_walk(repo.dirstate, matcher):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
162 lfile = splitstandin(standin)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
163 hash = readstandin(repo, lfile)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
164 lfdirstate.normallookup(lfile)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
165 try:
15553
e89385e4ef8d largefiles: file storage should be relative to repo, not relative to cwd
Mads Kiilerich <mads@kiilerich.com>
parents: 15548
diff changeset
166 if hash == hashfile(repo.wjoin(lfile)):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
167 lfdirstate.normal(lfile)
15548
f76584098c88 largefiles: fix 'hg clone . ../foo' OSError abort
Martin Geisler <mg@lazybytes.net>
parents: 15408
diff changeset
168 except OSError, err:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
169 if err.errno != errno.ENOENT:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
170 raise
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
171 return lfdirstate
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
172
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
173 def lfdirstate_status(lfdirstate, repo, rev):
15794
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
174 match = match_.always(repo.root, repo.getcwd())
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
175 s = lfdirstate.status(match, [], False, False, False)
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
176 unsure, modified, added, removed, missing, unknown, ignored, clean = s
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
177 for lfile in unsure:
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
178 if repo[rev][standin(lfile)].data().strip() != \
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
179 hashfile(repo.wjoin(lfile)):
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
180 modified.append(lfile)
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
181 else:
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
182 clean.append(lfile)
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
183 lfdirstate.normal(lfile)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
184 return (modified, added, removed, missing, unknown, ignored, clean)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
185
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
186 def listlfiles(repo, rev=None, matcher=None):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
187 '''return a list of largefiles in the working copy or the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
188 specified changeset'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
189
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
190 if matcher is None:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
191 matcher = getstandinmatcher(repo)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
192
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
193 # ignore unknown files in working directory
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
194 return [splitstandin(f)
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
195 for f in repo[rev].walk(matcher)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
196 if rev is not None or repo.dirstate[f] != '?']
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
197
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
198 def instore(repo, hash):
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
199 return os.path.exists(storepath(repo, hash))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
200
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
201 def storepath(repo, hash):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
202 return repo.join(os.path.join(longname, hash))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
203
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
204 def copyfromcache(repo, hash, filename):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
205 '''Copy the specified largefile from the repo or system cache to
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
206 filename in the repository. Return true on success or false if the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
207 file was not found in either cache (which should not happened:
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
208 this is meant to be called only after ensuring that the needed
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
209 largefile exists in the cache).'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
210 path = findfile(repo, hash)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
211 if path is None:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
212 return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
213 util.makedirs(os.path.dirname(repo.wjoin(filename)))
15570
0f208626d503 largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents: 15553
diff changeset
214 # The write may fail before the file is fully written, but we
0f208626d503 largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents: 15553
diff changeset
215 # don't use atomic writes in the working copy.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
216 shutil.copy(path, repo.wjoin(filename))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
217 return True
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
218
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
219 def copytostore(repo, rev, file, uploaded=False):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
220 hash = readstandin(repo, file)
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
221 if instore(repo, hash):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
222 return
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
223 copytostoreabsolute(repo, repo.wjoin(file), hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
224
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
225 def copytostoreabsolute(repo, file, hash):
15371
f26ed4ea46d8 largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents: 15350
diff changeset
226 util.makedirs(os.path.dirname(storepath(repo, hash)))
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
227 if inusercache(repo.ui, hash):
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
228 link(usercachepath(repo.ui, hash), storepath(repo, hash))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
229 else:
15571
809788118aa2 largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents: 15570
diff changeset
230 dst = util.atomictempfile(storepath(repo, hash))
15699
84e55467093c largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents: 15658
diff changeset
231 for chunk in util.filechunkiter(open(file, 'rb')):
15571
809788118aa2 largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents: 15570
diff changeset
232 dst.write(chunk)
809788118aa2 largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents: 15570
diff changeset
233 dst.close()
809788118aa2 largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents: 15570
diff changeset
234 util.copymode(file, storepath(repo, hash))
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
235 linktousercache(repo, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
236
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
237 def linktousercache(repo, hash):
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
238 path = usercachepath(repo.ui, hash)
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
239 if path:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
240 util.makedirs(os.path.dirname(path))
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
241 link(storepath(repo, hash), path)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
242
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
243 def getstandinmatcher(repo, pats=[], opts={}):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
244 '''Return a match object that applies pats to the standin directory'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
245 standindir = repo.pathto(shortname)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
246 if pats:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
247 # patterns supplied: search standin directory relative to current dir
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
248 cwd = repo.getcwd()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
249 if os.path.isabs(cwd):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
250 # cwd is an absolute path for hg -R <reponame>
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
251 # work relative to the repository root in this case
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
252 cwd = ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
253 pats = [os.path.join(standindir, cwd, pat) for pat in pats]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
254 elif os.path.isdir(standindir):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
255 # no patterns: relative to repo root
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
256 pats = [standindir]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
257 else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
258 # no patterns and no standin dir: return matcher that matches nothing
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
259 match = match_.match(repo.root, None, [], exact=True)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
260 match.matchfn = lambda f: False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
261 return match
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
262 return getmatcher(repo, pats, opts, showbad=False)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
263
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
264 def getmatcher(repo, pats=[], opts={}, showbad=True):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
265 '''Wrapper around scmutil.match() that adds showbad: if false,
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
266 neuter the match object's bad() method so it does not print any
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
267 warnings about missing files or directories.'''
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
268 match = scmutil.match(repo[None], pats, opts)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
269
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
270 if not showbad:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
271 match.bad = lambda f, msg: None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
272 return match
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
273
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
274 def composestandinmatcher(repo, rmatcher):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
275 '''Return a matcher that accepts standins corresponding to the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
276 files accepted by rmatcher. Pass the list of files in the matcher
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
277 as the paths specified by the user.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
278 smatcher = getstandinmatcher(repo, rmatcher.files())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
279 isstandin = smatcher.matchfn
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
280 def composed_matchfn(f):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
281 return isstandin(f) and rmatcher.matchfn(splitstandin(f))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
282 smatcher.matchfn = composed_matchfn
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
283
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
284 return smatcher
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
285
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
286 def standin(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
287 '''Return the repo-relative path to the standin for the specified big
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
288 file.'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
289 # Notes:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
290 # 1) Most callers want an absolute path, but _create_standin() needs
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
291 # it repo-relative so lfadd() can pass it to repo_add(). So leave
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
292 # it up to the caller to use repo.wjoin() to get an absolute path.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
293 # 2) Join with '/' because that's what dirstate always uses, even on
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
294 # Windows. Change existing separator to '/' first in case we are
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
295 # passed filenames from an external source (like the command line).
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
296 return shortname + '/' + filename.replace(os.sep, '/')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
297
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
298 def isstandin(filename):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
299 '''Return true if filename is a big file standin. filename must be
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
300 in Mercurial's internal form (slash-separated).'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
301 return filename.startswith(shortname + '/')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
302
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
303 def splitstandin(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
304 # Split on / because that's what dirstate always uses, even on Windows.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
305 # Change local separator to / first just in case we are passed filenames
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
306 # from an external source (like the command line).
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
307 bits = filename.replace(os.sep, '/').split('/', 1)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
308 if len(bits) == 2 and bits[0] == shortname:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
309 return bits[1]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
310 else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
311 return None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
312
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
313 def updatestandin(repo, standin):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
314 file = repo.wjoin(splitstandin(standin))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
315 if os.path.exists(file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
316 hash = hashfile(file)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
317 executable = getexecutable(file)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
318 writestandin(repo, standin, hash, executable)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
319
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
320 def readstandin(repo, filename, node=None):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
321 '''read hex hash from standin for filename at given node, or working
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
322 directory if no node is given'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
323 return repo[node][standin(filename)].data().strip()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
324
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
325 def writestandin(repo, standin, hash, executable):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
326 '''write hash to <repo.root>/<standin>'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
327 writehash(hash, repo.wjoin(standin), executable)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
328
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
329 def copyandhash(instream, outfile):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
330 '''Read bytes from instream (iterable) and write them to outfile,
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
331 computing the SHA-1 hash of the data along the way. Close outfile
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
332 when done and return the binary hash.'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
333 hasher = util.sha1('')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
334 for data in instream:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
335 hasher.update(data)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
336 outfile.write(data)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
337
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
338 # Blecch: closing a file that somebody else opened is rude and
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
339 # wrong. But it's so darn convenient and practical! After all,
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
340 # outfile was opened just to copy and hash.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
341 outfile.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
342
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
343 return hasher.digest()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
344
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
345 def hashrepofile(repo, file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
346 return hashfile(repo.wjoin(file))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
347
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
348 def hashfile(file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
349 if not os.path.exists(file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
350 return ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
351 hasher = util.sha1('')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
352 fd = open(file, 'rb')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
353 for data in blockstream(fd):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
354 hasher.update(data)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
355 fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
356 return hasher.hexdigest()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
357
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
358 class limitreader(object):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
359 def __init__(self, f, limit):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
360 self.f = f
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
361 self.limit = limit
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
362
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
363 def read(self, length):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
364 if self.limit == 0:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
365 return ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
366 length = length > self.limit and self.limit or length
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
367 self.limit -= length
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
368 return self.f.read(length)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
369
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
370 def close(self):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
371 pass
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
372
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
373 def blockstream(infile, blocksize=128 * 1024):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
374 """Generator that yields blocks of data from infile and closes infile."""
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
375 while True:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
376 data = infile.read(blocksize)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
377 if not data:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
378 break
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
379 yield data
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
380 # same blecch as copyandhash() above
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
381 infile.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
382
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
383 def readhash(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
384 rfile = open(filename, 'rb')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
385 hash = rfile.read(40)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
386 rfile.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
387 if len(hash) < 40:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
388 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
389 % (filename, len(hash)))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
390 return hash
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
391
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
392 def writehash(hash, filename, executable):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
393 util.makedirs(os.path.dirname(filename))
15574
c9328c829cd9 largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents: 15572
diff changeset
394 util.writefile(filename, hash + '\n')
c9328c829cd9 largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents: 15572
diff changeset
395 os.chmod(filename, getmode(executable))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
396
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
397 def getexecutable(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
398 mode = os.stat(filename).st_mode
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
399 return ((mode & stat.S_IXUSR) and
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
400 (mode & stat.S_IXGRP) and
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
401 (mode & stat.S_IXOTH))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
402
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
403 def getmode(executable):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
404 if executable:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
405 return 0755
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
406 else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
407 return 0644
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
408
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
409 def urljoin(first, second, *arg):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
410 def join(left, right):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
411 if not left.endswith('/'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
412 left += '/'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
413 if right.startswith('/'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
414 right = right[1:]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
415 return left + right
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
416
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
417 url = join(first, second)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
418 for a in arg:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
419 url = join(url, a)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
420 return url
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
421
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
422 def hexsha1(data):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
423 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
424 object data"""
15347
799e56609ef6 largefiles: use util.sha1() instead of hashlib.sha1() everywhere
Thomas Arendsen Hein <thomas@intevation.de>
parents: 15333
diff changeset
425 h = util.sha1()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
426 for chunk in util.filechunkiter(data):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
427 h.update(chunk)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
428 return h.hexdigest()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
429
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
430 def httpsendfile(ui, filename):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
431 return httpconnection.httpsendfile(ui, filename, 'rb')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
432
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
433 def unixpath(path):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
434 '''Return a version of path normalized for use with the lfdirstate.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
435 return os.path.normpath(path).replace(os.sep, '/')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
436
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
437 def islfilesrepo(repo):
15170
c1a4a3220711 largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents: 15169
diff changeset
438 return ('largefiles' in repo.requirements and
15319
9da7e96cd5c2 largefiles: remove redundant any_ function
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15317
diff changeset
439 util.any(shortname + '/' in f[0] for f in repo.store.datafiles()))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
440
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
441 def mkstemp(repo, prefix):
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
442 '''Returns a file descriptor and a filename corresponding to a temporary
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
443 file in the repo's largefiles store.'''
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
444 path = repo.join(longname)
15392
d7bfbc92a1c0 util: add a doctest for empty sha() calls
Matt Mackall <mpm@selenic.com>
parents: 15391
diff changeset
445 util.makedirs(path)
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
446 return tempfile.mkstemp(prefix=prefix, dir=path)
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15371
diff changeset
447
15333
f37b71fec602 largefiles: py2.4 doesn't have BaseException
Matt Mackall <mpm@selenic.com>
parents: 15320
diff changeset
448 class storeprotonotcapable(Exception):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
449 def __init__(self, storetypes):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
450 self.storetypes = storetypes