Mercurial > hg
annotate hgext/largefiles/lfutil.py @ 15663:9036c7d106bf stable
largefiles: handle merges between normal files and largefiles (issue3084)
The largefiles extension prevents users from adding a normal file
named 'foo' if there is already a largefile with the same name.
However, there was a loop-hole: when merging, it was possible to bring
in a normal file named 'foo' while also having a '.hglf/foo' file.
This patch fixes this by extending the manifest merge to deal with
these kinds of conflicts. If there is a normal file 'foo' in the
working copy, and the other parent brings in a '.hglf/foo' file, then
the user will be prompted to keep the normal file or the largefile.
Likewise for the symmetric case where a normal file is brought in via
the second parent. The prompt looks like this:
$ hg merge
foo has been turned into a largefile
use (l)argefile or keep as (n)ormal file?
After the merge, either the '.hglf/foo' file or the 'foo' file will
have been deleted. This would cause status to return output like:
$ hg status
M foo
R foo
To fix this, the lfiles_repo.status method is changed so that a
removed normal file isn't shown if there is largefile with the same
name, and vice versa for largefiles.
author | Martin Geisler <mg@aragost.com> |
---|---|
date | Fri, 09 Dec 2011 17:35:00 +0100 |
parents | 971c55ce03b8 |
children | c7b0bedbb07a 84e55467093c |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''largefiles utility code: must not import other modules in this package.''' | |
10 | |
11 import os | |
12 import errno | |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
13 import platform |
15168 | 14 import shutil |
15 import stat | |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
16 import tempfile |
15168 | 17 |
15226
2223ea21c98f
largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
18 from mercurial import dirstate, httpconnection, match as match_, util, scmutil |
15168 | 19 from mercurial.i18n import _ |
20 | |
21 shortname = '.hglf' | |
22 longname = 'largefiles' | |
23 | |
24 | |
25 # -- Portability wrappers ---------------------------------------------- | |
26 | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
27 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
28 return dirstate.walk(matcher, [], unknown, ignored) |
15168 | 29 |
30 def repo_add(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
31 add = repo[None].add |
15168 | 32 return add(list) |
33 | |
34 def repo_remove(repo, list, unlink=False): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
35 def remove(list, unlink): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
36 wlock = repo.wlock() |
15168 | 37 try: |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
38 if unlink: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
39 for f in list: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
40 try: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
41 util.unlinkpath(repo.wjoin(f)) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
42 except OSError, inst: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
43 if inst.errno != errno.ENOENT: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
44 raise |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
45 repo[None].forget(list) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
46 finally: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
47 wlock.release() |
15168 | 48 return remove(list, unlink=unlink) |
49 | |
50 def repo_forget(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
51 forget = repo[None].forget |
15168 | 52 return forget(list) |
53 | |
54 def findoutgoing(repo, remote, force): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
55 from mercurial import discovery |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
56 common, _anyinc, _heads = discovery.findcommonincoming(repo, |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
57 remote, force=force) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
58 return repo.changelog.findmissing(common) |
15168 | 59 |
60 # -- Private worker functions ------------------------------------------ | |
61 | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
62 def getminsize(ui, assumelfiles, opt, default=10): |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
63 lfsize = opt |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
64 if not lfsize and assumelfiles: |
15304
9aa9d4bb3d88
largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents:
15255
diff
changeset
|
65 lfsize = ui.config(longname, 'minsize', default=default) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
66 if lfsize: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
67 try: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
68 lfsize = float(lfsize) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
69 except ValueError: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
70 raise util.Abort(_('largefiles: size must be number (not %s)\n') |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
71 % lfsize) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
72 if lfsize is None: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
73 raise util.Abort(_('minimum size for largefiles must be specified')) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
74 return lfsize |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
75 |
15168 | 76 def link(src, dest): |
77 try: | |
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
78 util.oslink(src, dest) |
15168 | 79 except OSError: |
15572
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
80 # if hardlinks fail, fallback on atomic copy |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
81 dst = util.atomictempfile(dest) |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
82 for chunk in util.filechunkiter(open(src)): |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
83 dst.write(chunk) |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
84 dst.close() |
15168 | 85 os.chmod(dest, os.stat(src).st_mode) |
86 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
87 def usercachepath(ui, hash): |
15350
8b8dd13295db
largefiles: use ui.configpath() where appropriate
Greg Ward <greg@gerg.ca>
parents:
15349
diff
changeset
|
88 path = ui.configpath(longname, 'usercache', None) |
15168 | 89 if path: |
90 path = os.path.join(path, hash) | |
91 else: | |
92 if os.name == 'nt': | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
93 appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
94 if appdata: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
95 path = os.path.join(appdata, longname, hash) |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
96 elif platform.system() == 'Darwin': |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
97 home = os.getenv('HOME') |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
98 if home: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
99 path = os.path.join(home, 'Library', 'Caches', |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
100 longname, hash) |
15168 | 101 elif os.name == 'posix': |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
102 path = os.getenv('XDG_CACHE_HOME') |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
103 if path: |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
104 path = os.path.join(path, longname, hash) |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
105 else: |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
106 home = os.getenv('HOME') |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
107 if home: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
108 path = os.path.join(home, '.cache', longname, hash) |
15168 | 109 else: |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
110 raise util.Abort(_('unknown operating system: %s\n') % os.name) |
15168 | 111 return path |
112 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
113 def inusercache(ui, hash): |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
114 path = usercachepath(ui, hash) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
115 return path and os.path.exists(path) |
15168 | 116 |
117 def findfile(repo, hash): | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
118 if instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
119 repo.ui.note(_('Found %s in store\n') % hash) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
120 elif inusercache(repo.ui, hash): |
15168 | 121 repo.ui.note(_('Found %s in system cache\n') % hash) |
15408
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
122 path = storepath(repo, hash) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
123 util.makedirs(os.path.dirname(path)) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
124 link(usercachepath(repo.ui, hash), path) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
125 else: |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
126 return None |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
127 return storepath(repo, hash) |
15168 | 128 |
129 class largefiles_dirstate(dirstate.dirstate): | |
130 def __getitem__(self, key): | |
131 return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | |
132 def normal(self, f): | |
133 return super(largefiles_dirstate, self).normal(unixpath(f)) | |
134 def remove(self, f): | |
135 return super(largefiles_dirstate, self).remove(unixpath(f)) | |
136 def add(self, f): | |
137 return super(largefiles_dirstate, self).add(unixpath(f)) | |
138 def drop(self, f): | |
139 return super(largefiles_dirstate, self).drop(unixpath(f)) | |
140 def forget(self, f): | |
141 return super(largefiles_dirstate, self).forget(unixpath(f)) | |
142 | |
143 def openlfdirstate(ui, repo): | |
144 ''' | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
145 Return a dirstate object that tracks largefiles: i.e. its root is |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
146 the repo root, but it is saved in .hg/largefiles/dirstate. |
15168 | 147 ''' |
148 admin = repo.join(longname) | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
149 opener = scmutil.opener(admin) |
15349
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
150 lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
151 repo.dirstate._validate) |
15168 | 152 |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
153 # If the largefiles dirstate does not exist, populate and create |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
154 # it. This ensures that we create it on the first meaningful |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
155 # largefiles operation in a new clone. It also gives us an easy |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
156 # way to forcibly rebuild largefiles state: |
15168 | 157 # rm .hg/largefiles/dirstate && hg status |
158 # Or even, if things are really messed up: | |
159 # rm -rf .hg/largefiles && hg status | |
160 if not os.path.exists(os.path.join(admin, 'dirstate')): | |
161 util.makedirs(admin) | |
162 matcher = getstandinmatcher(repo) | |
163 for standin in dirstate_walk(repo.dirstate, matcher): | |
164 lfile = splitstandin(standin) | |
165 hash = readstandin(repo, lfile) | |
166 lfdirstate.normallookup(lfile) | |
167 try: | |
15553
e89385e4ef8d
largefiles: file storage should be relative to repo, not relative to cwd
Mads Kiilerich <mads@kiilerich.com>
parents:
15548
diff
changeset
|
168 if hash == hashfile(repo.wjoin(lfile)): |
15168 | 169 lfdirstate.normal(lfile) |
15548
f76584098c88
largefiles: fix 'hg clone . ../foo' OSError abort
Martin Geisler <mg@lazybytes.net>
parents:
15408
diff
changeset
|
170 except OSError, err: |
15168 | 171 if err.errno != errno.ENOENT: |
172 raise | |
173 | |
174 lfdirstate.write() | |
175 | |
176 return lfdirstate | |
177 | |
178 def lfdirstate_status(lfdirstate, repo, rev): | |
179 wlock = repo.wlock() | |
180 try: | |
181 match = match_.always(repo.root, repo.getcwd()) | |
182 s = lfdirstate.status(match, [], False, False, False) | |
183 unsure, modified, added, removed, missing, unknown, ignored, clean = s | |
184 for lfile in unsure: | |
185 if repo[rev][standin(lfile)].data().strip() != \ | |
186 hashfile(repo.wjoin(lfile)): | |
187 modified.append(lfile) | |
188 else: | |
189 clean.append(lfile) | |
190 lfdirstate.normal(lfile) | |
191 lfdirstate.write() | |
192 finally: | |
193 wlock.release() | |
194 return (modified, added, removed, missing, unknown, ignored, clean) | |
195 | |
196 def listlfiles(repo, rev=None, matcher=None): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
197 '''return a list of largefiles in the working copy or the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
198 specified changeset''' |
15168 | 199 |
200 if matcher is None: | |
201 matcher = getstandinmatcher(repo) | |
202 | |
203 # ignore unknown files in working directory | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
204 return [splitstandin(f) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
205 for f in repo[rev].walk(matcher) |
15168 | 206 if rev is not None or repo.dirstate[f] != '?'] |
207 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
208 def instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
209 return os.path.exists(storepath(repo, hash)) |
15168 | 210 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
211 def storepath(repo, hash): |
15168 | 212 return repo.join(os.path.join(longname, hash)) |
213 | |
214 def copyfromcache(repo, hash, filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
215 '''Copy the specified largefile from the repo or system cache to |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
216 filename in the repository. Return true on success or false if the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
217 file was not found in either cache (which should not happened: |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
218 this is meant to be called only after ensuring that the needed |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
219 largefile exists in the cache).''' |
15168 | 220 path = findfile(repo, hash) |
221 if path is None: | |
222 return False | |
223 util.makedirs(os.path.dirname(repo.wjoin(filename))) | |
15570
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
224 # The write may fail before the file is fully written, but we |
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
225 # don't use atomic writes in the working copy. |
15168 | 226 shutil.copy(path, repo.wjoin(filename)) |
227 return True | |
228 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
229 def copytostore(repo, rev, file, uploaded=False): |
15168 | 230 hash = readstandin(repo, file) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
231 if instore(repo, hash): |
15168 | 232 return |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
233 copytostoreabsolute(repo, repo.wjoin(file), hash) |
15168 | 234 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
235 def copytostoreabsolute(repo, file, hash): |
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15350
diff
changeset
|
236 util.makedirs(os.path.dirname(storepath(repo, hash))) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
237 if inusercache(repo.ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
238 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
15168 | 239 else: |
15571
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
240 dst = util.atomictempfile(storepath(repo, hash)) |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
241 for chunk in util.filechunkiter(open(file)): |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
242 dst.write(chunk) |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
243 dst.close() |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
244 util.copymode(file, storepath(repo, hash)) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
245 linktousercache(repo, hash) |
15168 | 246 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
247 def linktousercache(repo, hash): |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
248 path = usercachepath(repo.ui, hash) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
249 if path: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
250 util.makedirs(os.path.dirname(path)) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
251 link(storepath(repo, hash), path) |
15168 | 252 |
253 def getstandinmatcher(repo, pats=[], opts={}): | |
254 '''Return a match object that applies pats to the standin directory''' | |
255 standindir = repo.pathto(shortname) | |
256 if pats: | |
257 # patterns supplied: search standin directory relative to current dir | |
258 cwd = repo.getcwd() | |
259 if os.path.isabs(cwd): | |
260 # cwd is an absolute path for hg -R <reponame> | |
261 # work relative to the repository root in this case | |
262 cwd = '' | |
263 pats = [os.path.join(standindir, cwd, pat) for pat in pats] | |
264 elif os.path.isdir(standindir): | |
265 # no patterns: relative to repo root | |
266 pats = [standindir] | |
267 else: | |
268 # no patterns and no standin dir: return matcher that matches nothing | |
269 match = match_.match(repo.root, None, [], exact=True) | |
270 match.matchfn = lambda f: False | |
271 return match | |
272 return getmatcher(repo, pats, opts, showbad=False) | |
273 | |
274 def getmatcher(repo, pats=[], opts={}, showbad=True): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
275 '''Wrapper around scmutil.match() that adds showbad: if false, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
276 neuter the match object's bad() method so it does not print any |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
277 warnings about missing files or directories.''' |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
278 match = scmutil.match(repo[None], pats, opts) |
15168 | 279 |
280 if not showbad: | |
281 match.bad = lambda f, msg: None | |
282 return match | |
283 | |
284 def composestandinmatcher(repo, rmatcher): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
285 '''Return a matcher that accepts standins corresponding to the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
286 files accepted by rmatcher. Pass the list of files in the matcher |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
287 as the paths specified by the user.''' |
15168 | 288 smatcher = getstandinmatcher(repo, rmatcher.files()) |
289 isstandin = smatcher.matchfn | |
290 def composed_matchfn(f): | |
291 return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | |
292 smatcher.matchfn = composed_matchfn | |
293 | |
294 return smatcher | |
295 | |
296 def standin(filename): | |
297 '''Return the repo-relative path to the standin for the specified big | |
298 file.''' | |
299 # Notes: | |
300 # 1) Most callers want an absolute path, but _create_standin() needs | |
301 # it repo-relative so lfadd() can pass it to repo_add(). So leave | |
302 # it up to the caller to use repo.wjoin() to get an absolute path. | |
303 # 2) Join with '/' because that's what dirstate always uses, even on | |
304 # Windows. Change existing separator to '/' first in case we are | |
305 # passed filenames from an external source (like the command line). | |
306 return shortname + '/' + filename.replace(os.sep, '/') | |
307 | |
308 def isstandin(filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
309 '''Return true if filename is a big file standin. filename must be |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
310 in Mercurial's internal form (slash-separated).''' |
15168 | 311 return filename.startswith(shortname + '/') |
312 | |
313 def splitstandin(filename): | |
314 # Split on / because that's what dirstate always uses, even on Windows. | |
315 # Change local separator to / first just in case we are passed filenames | |
316 # from an external source (like the command line). | |
317 bits = filename.replace(os.sep, '/').split('/', 1) | |
318 if len(bits) == 2 and bits[0] == shortname: | |
319 return bits[1] | |
320 else: | |
321 return None | |
322 | |
323 def updatestandin(repo, standin): | |
324 file = repo.wjoin(splitstandin(standin)) | |
325 if os.path.exists(file): | |
326 hash = hashfile(file) | |
327 executable = getexecutable(file) | |
328 writestandin(repo, standin, hash, executable) | |
329 | |
330 def readstandin(repo, filename, node=None): | |
331 '''read hex hash from standin for filename at given node, or working | |
332 directory if no node is given''' | |
333 return repo[node][standin(filename)].data().strip() | |
334 | |
335 def writestandin(repo, standin, hash, executable): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
336 '''write hash to <repo.root>/<standin>''' |
15168 | 337 writehash(hash, repo.wjoin(standin), executable) |
338 | |
339 def copyandhash(instream, outfile): | |
340 '''Read bytes from instream (iterable) and write them to outfile, | |
341 computing the SHA-1 hash of the data along the way. Close outfile | |
342 when done and return the binary hash.''' | |
343 hasher = util.sha1('') | |
344 for data in instream: | |
345 hasher.update(data) | |
346 outfile.write(data) | |
347 | |
348 # Blecch: closing a file that somebody else opened is rude and | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
349 # wrong. But it's so darn convenient and practical! After all, |
15168 | 350 # outfile was opened just to copy and hash. |
351 outfile.close() | |
352 | |
353 return hasher.digest() | |
354 | |
355 def hashrepofile(repo, file): | |
356 return hashfile(repo.wjoin(file)) | |
357 | |
358 def hashfile(file): | |
359 if not os.path.exists(file): | |
360 return '' | |
361 hasher = util.sha1('') | |
362 fd = open(file, 'rb') | |
363 for data in blockstream(fd): | |
364 hasher.update(data) | |
365 fd.close() | |
366 return hasher.hexdigest() | |
367 | |
368 class limitreader(object): | |
369 def __init__(self, f, limit): | |
370 self.f = f | |
371 self.limit = limit | |
372 | |
373 def read(self, length): | |
374 if self.limit == 0: | |
375 return '' | |
376 length = length > self.limit and self.limit or length | |
377 self.limit -= length | |
378 return self.f.read(length) | |
379 | |
380 def close(self): | |
381 pass | |
382 | |
383 def blockstream(infile, blocksize=128 * 1024): | |
384 """Generator that yields blocks of data from infile and closes infile.""" | |
385 while True: | |
386 data = infile.read(blocksize) | |
387 if not data: | |
388 break | |
389 yield data | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
390 # same blecch as copyandhash() above |
15168 | 391 infile.close() |
392 | |
393 def readhash(filename): | |
394 rfile = open(filename, 'rb') | |
395 hash = rfile.read(40) | |
396 rfile.close() | |
397 if len(hash) < 40: | |
398 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') | |
399 % (filename, len(hash))) | |
400 return hash | |
401 | |
402 def writehash(hash, filename, executable): | |
403 util.makedirs(os.path.dirname(filename)) | |
404 if os.path.exists(filename): | |
405 os.unlink(filename) | |
406 wfile = open(filename, 'wb') | |
407 | |
408 try: | |
409 wfile.write(hash) | |
410 wfile.write('\n') | |
411 finally: | |
412 wfile.close() | |
413 if os.path.exists(filename): | |
414 os.chmod(filename, getmode(executable)) | |
415 | |
416 def getexecutable(filename): | |
417 mode = os.stat(filename).st_mode | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
418 return ((mode & stat.S_IXUSR) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
419 (mode & stat.S_IXGRP) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
420 (mode & stat.S_IXOTH)) |
15168 | 421 |
422 def getmode(executable): | |
423 if executable: | |
424 return 0755 | |
425 else: | |
426 return 0644 | |
427 | |
428 def urljoin(first, second, *arg): | |
429 def join(left, right): | |
430 if not left.endswith('/'): | |
431 left += '/' | |
432 if right.startswith('/'): | |
433 right = right[1:] | |
434 return left + right | |
435 | |
436 url = join(first, second) | |
437 for a in arg: | |
438 url = join(url, a) | |
439 return url | |
440 | |
441 def hexsha1(data): | |
442 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | |
443 object data""" | |
15347
799e56609ef6
largefiles: use util.sha1() instead of hashlib.sha1() everywhere
Thomas Arendsen Hein <thomas@intevation.de>
parents:
15333
diff
changeset
|
444 h = util.sha1() |
15168 | 445 for chunk in util.filechunkiter(data): |
446 h.update(chunk) | |
447 return h.hexdigest() | |
448 | |
449 def httpsendfile(ui, filename): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
450 return httpconnection.httpsendfile(ui, filename, 'rb') |
15168 | 451 |
452 def unixpath(path): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
453 '''Return a version of path normalized for use with the lfdirstate.''' |
15168 | 454 return os.path.normpath(path).replace(os.sep, '/') |
455 | |
456 def islfilesrepo(repo): | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
457 return ('largefiles' in repo.requirements and |
15319
9da7e96cd5c2
largefiles: remove redundant any_ function
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15317
diff
changeset
|
458 util.any(shortname + '/' in f[0] for f in repo.store.datafiles())) |
15168 | 459 |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
460 def mkstemp(repo, prefix): |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
461 '''Returns a file descriptor and a filename corresponding to a temporary |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
462 file in the repo's largefiles store.''' |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
463 path = repo.join(longname) |
15392
d7bfbc92a1c0
util: add a doctest for empty sha() calls
Matt Mackall <mpm@selenic.com>
parents:
15391
diff
changeset
|
464 util.makedirs(path) |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
465 return tempfile.mkstemp(prefix=prefix, dir=path) |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
466 |
15333
f37b71fec602
largefiles: py2.4 doesn't have BaseException
Matt Mackall <mpm@selenic.com>
parents:
15320
diff
changeset
|
467 class storeprotonotcapable(Exception): |
15168 | 468 def __init__(self, storetypes): |
469 self.storetypes = storetypes |