Mercurial > hg
annotate hgext/largefiles/lfutil.py @ 15572:926bc23d0b6a stable
largefiles: copy files into .hg/largefiles atomically
Copying from the user cache into .hg/largefiles could fail halfway
though with a partially written file.
author | Martin Geisler <mg@aragost.com> |
---|---|
date | Thu, 24 Nov 2011 18:13:18 +0100 |
parents | 809788118aa2 |
children | c9328c829cd9 971c55ce03b8 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''largefiles utility code: must not import other modules in this package.''' | |
10 | |
11 import os | |
12 import errno | |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
13 import platform |
15168 | 14 import shutil |
15 import stat | |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
16 import tempfile |
15168 | 17 |
15226
2223ea21c98f
largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
18 from mercurial import dirstate, httpconnection, match as match_, util, scmutil |
15168 | 19 from mercurial.i18n import _ |
20 | |
21 shortname = '.hglf' | |
22 longname = 'largefiles' | |
23 | |
24 | |
25 # -- Portability wrappers ---------------------------------------------- | |
26 | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
27 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
28 return dirstate.walk(matcher, [], unknown, ignored) |
15168 | 29 |
30 def repo_add(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
31 add = repo[None].add |
15168 | 32 return add(list) |
33 | |
34 def repo_remove(repo, list, unlink=False): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
35 def remove(list, unlink): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
36 wlock = repo.wlock() |
15168 | 37 try: |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
38 if unlink: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
39 for f in list: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
40 try: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
41 util.unlinkpath(repo.wjoin(f)) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
42 except OSError, inst: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
43 if inst.errno != errno.ENOENT: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
44 raise |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
45 repo[None].forget(list) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
46 finally: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
47 wlock.release() |
15168 | 48 return remove(list, unlink=unlink) |
49 | |
50 def repo_forget(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
51 forget = repo[None].forget |
15168 | 52 return forget(list) |
53 | |
54 def findoutgoing(repo, remote, force): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
55 from mercurial import discovery |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
56 common, _anyinc, _heads = discovery.findcommonincoming(repo, |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
57 remote, force=force) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
58 return repo.changelog.findmissing(common) |
15168 | 59 |
60 # -- Private worker functions ------------------------------------------ | |
61 | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
62 def getminsize(ui, assumelfiles, opt, default=10): |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
63 lfsize = opt |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
64 if not lfsize and assumelfiles: |
15304
9aa9d4bb3d88
largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents:
15255
diff
changeset
|
65 lfsize = ui.config(longname, 'minsize', default=default) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
66 if lfsize: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
67 try: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
68 lfsize = float(lfsize) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
69 except ValueError: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
70 raise util.Abort(_('largefiles: size must be number (not %s)\n') |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
71 % lfsize) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
72 if lfsize is None: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
73 raise util.Abort(_('minimum size for largefiles must be specified')) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
74 return lfsize |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
75 |
15168 | 76 def link(src, dest): |
77 try: | |
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
78 util.oslink(src, dest) |
15168 | 79 except OSError: |
15572
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
80 # if hardlinks fail, fallback on atomic copy |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
81 dst = util.atomictempfile(dest) |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
82 for chunk in util.filechunkiter(open(src)): |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
83 dst.write(chunk) |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
84 dst.close() |
15168 | 85 os.chmod(dest, os.stat(src).st_mode) |
86 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
87 def usercachepath(ui, hash): |
15350
8b8dd13295db
largefiles: use ui.configpath() where appropriate
Greg Ward <greg@gerg.ca>
parents:
15349
diff
changeset
|
88 path = ui.configpath(longname, 'usercache', None) |
15168 | 89 if path: |
90 path = os.path.join(path, hash) | |
91 else: | |
92 if os.name == 'nt': | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
93 appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
94 path = os.path.join(appdata, longname, hash) |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
95 elif platform.system() == 'Darwin': |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
96 path = os.path.join(os.getenv('HOME'), 'Library', 'Caches', |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
97 longname, hash) |
15168 | 98 elif os.name == 'posix': |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
99 path = os.getenv('XDG_CACHE_HOME') |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
100 if path: |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
101 path = os.path.join(path, longname, hash) |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
102 else: |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
103 path = os.path.join(os.getenv('HOME'), '.cache', longname, hash) |
15168 | 104 else: |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
105 raise util.Abort(_('unknown operating system: %s\n') % os.name) |
15168 | 106 return path |
107 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
108 def inusercache(ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
109 return os.path.exists(usercachepath(ui, hash)) |
15168 | 110 |
111 def findfile(repo, hash): | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
112 if instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
113 repo.ui.note(_('Found %s in store\n') % hash) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
114 elif inusercache(repo.ui, hash): |
15168 | 115 repo.ui.note(_('Found %s in system cache\n') % hash) |
15408
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
116 path = storepath(repo, hash) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
117 util.makedirs(os.path.dirname(path)) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
118 link(usercachepath(repo.ui, hash), path) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
119 else: |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
120 return None |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
121 return storepath(repo, hash) |
15168 | 122 |
123 class largefiles_dirstate(dirstate.dirstate): | |
124 def __getitem__(self, key): | |
125 return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | |
126 def normal(self, f): | |
127 return super(largefiles_dirstate, self).normal(unixpath(f)) | |
128 def remove(self, f): | |
129 return super(largefiles_dirstate, self).remove(unixpath(f)) | |
130 def add(self, f): | |
131 return super(largefiles_dirstate, self).add(unixpath(f)) | |
132 def drop(self, f): | |
133 return super(largefiles_dirstate, self).drop(unixpath(f)) | |
134 def forget(self, f): | |
135 return super(largefiles_dirstate, self).forget(unixpath(f)) | |
136 | |
137 def openlfdirstate(ui, repo): | |
138 ''' | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
139 Return a dirstate object that tracks largefiles: i.e. its root is |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
140 the repo root, but it is saved in .hg/largefiles/dirstate. |
15168 | 141 ''' |
142 admin = repo.join(longname) | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
143 opener = scmutil.opener(admin) |
15349
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
144 lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
145 repo.dirstate._validate) |
15168 | 146 |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
147 # If the largefiles dirstate does not exist, populate and create |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
148 # it. This ensures that we create it on the first meaningful |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
149 # largefiles operation in a new clone. It also gives us an easy |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
150 # way to forcibly rebuild largefiles state: |
15168 | 151 # rm .hg/largefiles/dirstate && hg status |
152 # Or even, if things are really messed up: | |
153 # rm -rf .hg/largefiles && hg status | |
154 if not os.path.exists(os.path.join(admin, 'dirstate')): | |
155 util.makedirs(admin) | |
156 matcher = getstandinmatcher(repo) | |
157 for standin in dirstate_walk(repo.dirstate, matcher): | |
158 lfile = splitstandin(standin) | |
159 hash = readstandin(repo, lfile) | |
160 lfdirstate.normallookup(lfile) | |
161 try: | |
15553
e89385e4ef8d
largefiles: file storage should be relative to repo, not relative to cwd
Mads Kiilerich <mads@kiilerich.com>
parents:
15548
diff
changeset
|
162 if hash == hashfile(repo.wjoin(lfile)): |
15168 | 163 lfdirstate.normal(lfile) |
15548
f76584098c88
largefiles: fix 'hg clone . ../foo' OSError abort
Martin Geisler <mg@lazybytes.net>
parents:
15408
diff
changeset
|
164 except OSError, err: |
15168 | 165 if err.errno != errno.ENOENT: |
166 raise | |
167 | |
168 lfdirstate.write() | |
169 | |
170 return lfdirstate | |
171 | |
172 def lfdirstate_status(lfdirstate, repo, rev): | |
173 wlock = repo.wlock() | |
174 try: | |
175 match = match_.always(repo.root, repo.getcwd()) | |
176 s = lfdirstate.status(match, [], False, False, False) | |
177 unsure, modified, added, removed, missing, unknown, ignored, clean = s | |
178 for lfile in unsure: | |
179 if repo[rev][standin(lfile)].data().strip() != \ | |
180 hashfile(repo.wjoin(lfile)): | |
181 modified.append(lfile) | |
182 else: | |
183 clean.append(lfile) | |
184 lfdirstate.normal(lfile) | |
185 lfdirstate.write() | |
186 finally: | |
187 wlock.release() | |
188 return (modified, added, removed, missing, unknown, ignored, clean) | |
189 | |
190 def listlfiles(repo, rev=None, matcher=None): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
191 '''return a list of largefiles in the working copy or the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
192 specified changeset''' |
15168 | 193 |
194 if matcher is None: | |
195 matcher = getstandinmatcher(repo) | |
196 | |
197 # ignore unknown files in working directory | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
198 return [splitstandin(f) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
199 for f in repo[rev].walk(matcher) |
15168 | 200 if rev is not None or repo.dirstate[f] != '?'] |
201 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
202 def instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
203 return os.path.exists(storepath(repo, hash)) |
15168 | 204 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
205 def storepath(repo, hash): |
15168 | 206 return repo.join(os.path.join(longname, hash)) |
207 | |
208 def copyfromcache(repo, hash, filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
209 '''Copy the specified largefile from the repo or system cache to |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
210 filename in the repository. Return true on success or false if the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
211 file was not found in either cache (which should not happened: |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
212 this is meant to be called only after ensuring that the needed |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
213 largefile exists in the cache).''' |
15168 | 214 path = findfile(repo, hash) |
215 if path is None: | |
216 return False | |
217 util.makedirs(os.path.dirname(repo.wjoin(filename))) | |
15570
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
218 # The write may fail before the file is fully written, but we |
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
219 # don't use atomic writes in the working copy. |
15168 | 220 shutil.copy(path, repo.wjoin(filename)) |
221 return True | |
222 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
223 def copytostore(repo, rev, file, uploaded=False): |
15168 | 224 hash = readstandin(repo, file) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
225 if instore(repo, hash): |
15168 | 226 return |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
227 copytostoreabsolute(repo, repo.wjoin(file), hash) |
15168 | 228 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
229 def copytostoreabsolute(repo, file, hash): |
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15350
diff
changeset
|
230 util.makedirs(os.path.dirname(storepath(repo, hash))) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
231 if inusercache(repo.ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
232 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
15168 | 233 else: |
15571
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
234 dst = util.atomictempfile(storepath(repo, hash)) |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
235 for chunk in util.filechunkiter(open(file)): |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
236 dst.write(chunk) |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
237 dst.close() |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
238 util.copymode(file, storepath(repo, hash)) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
239 linktousercache(repo, hash) |
15168 | 240 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
241 def linktousercache(repo, hash): |
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15350
diff
changeset
|
242 util.makedirs(os.path.dirname(usercachepath(repo.ui, hash))) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
243 link(storepath(repo, hash), usercachepath(repo.ui, hash)) |
15168 | 244 |
245 def getstandinmatcher(repo, pats=[], opts={}): | |
246 '''Return a match object that applies pats to the standin directory''' | |
247 standindir = repo.pathto(shortname) | |
248 if pats: | |
249 # patterns supplied: search standin directory relative to current dir | |
250 cwd = repo.getcwd() | |
251 if os.path.isabs(cwd): | |
252 # cwd is an absolute path for hg -R <reponame> | |
253 # work relative to the repository root in this case | |
254 cwd = '' | |
255 pats = [os.path.join(standindir, cwd, pat) for pat in pats] | |
256 elif os.path.isdir(standindir): | |
257 # no patterns: relative to repo root | |
258 pats = [standindir] | |
259 else: | |
260 # no patterns and no standin dir: return matcher that matches nothing | |
261 match = match_.match(repo.root, None, [], exact=True) | |
262 match.matchfn = lambda f: False | |
263 return match | |
264 return getmatcher(repo, pats, opts, showbad=False) | |
265 | |
266 def getmatcher(repo, pats=[], opts={}, showbad=True): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
267 '''Wrapper around scmutil.match() that adds showbad: if false, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
268 neuter the match object's bad() method so it does not print any |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
269 warnings about missing files or directories.''' |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
270 match = scmutil.match(repo[None], pats, opts) |
15168 | 271 |
272 if not showbad: | |
273 match.bad = lambda f, msg: None | |
274 return match | |
275 | |
276 def composestandinmatcher(repo, rmatcher): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
277 '''Return a matcher that accepts standins corresponding to the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
278 files accepted by rmatcher. Pass the list of files in the matcher |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
279 as the paths specified by the user.''' |
15168 | 280 smatcher = getstandinmatcher(repo, rmatcher.files()) |
281 isstandin = smatcher.matchfn | |
282 def composed_matchfn(f): | |
283 return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | |
284 smatcher.matchfn = composed_matchfn | |
285 | |
286 return smatcher | |
287 | |
288 def standin(filename): | |
289 '''Return the repo-relative path to the standin for the specified big | |
290 file.''' | |
291 # Notes: | |
292 # 1) Most callers want an absolute path, but _create_standin() needs | |
293 # it repo-relative so lfadd() can pass it to repo_add(). So leave | |
294 # it up to the caller to use repo.wjoin() to get an absolute path. | |
295 # 2) Join with '/' because that's what dirstate always uses, even on | |
296 # Windows. Change existing separator to '/' first in case we are | |
297 # passed filenames from an external source (like the command line). | |
298 return shortname + '/' + filename.replace(os.sep, '/') | |
299 | |
300 def isstandin(filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
301 '''Return true if filename is a big file standin. filename must be |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
302 in Mercurial's internal form (slash-separated).''' |
15168 | 303 return filename.startswith(shortname + '/') |
304 | |
305 def splitstandin(filename): | |
306 # Split on / because that's what dirstate always uses, even on Windows. | |
307 # Change local separator to / first just in case we are passed filenames | |
308 # from an external source (like the command line). | |
309 bits = filename.replace(os.sep, '/').split('/', 1) | |
310 if len(bits) == 2 and bits[0] == shortname: | |
311 return bits[1] | |
312 else: | |
313 return None | |
314 | |
315 def updatestandin(repo, standin): | |
316 file = repo.wjoin(splitstandin(standin)) | |
317 if os.path.exists(file): | |
318 hash = hashfile(file) | |
319 executable = getexecutable(file) | |
320 writestandin(repo, standin, hash, executable) | |
321 | |
322 def readstandin(repo, filename, node=None): | |
323 '''read hex hash from standin for filename at given node, or working | |
324 directory if no node is given''' | |
325 return repo[node][standin(filename)].data().strip() | |
326 | |
327 def writestandin(repo, standin, hash, executable): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
328 '''write hash to <repo.root>/<standin>''' |
15168 | 329 writehash(hash, repo.wjoin(standin), executable) |
330 | |
331 def copyandhash(instream, outfile): | |
332 '''Read bytes from instream (iterable) and write them to outfile, | |
333 computing the SHA-1 hash of the data along the way. Close outfile | |
334 when done and return the binary hash.''' | |
335 hasher = util.sha1('') | |
336 for data in instream: | |
337 hasher.update(data) | |
338 outfile.write(data) | |
339 | |
340 # Blecch: closing a file that somebody else opened is rude and | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
341 # wrong. But it's so darn convenient and practical! After all, |
15168 | 342 # outfile was opened just to copy and hash. |
343 outfile.close() | |
344 | |
345 return hasher.digest() | |
346 | |
347 def hashrepofile(repo, file): | |
348 return hashfile(repo.wjoin(file)) | |
349 | |
350 def hashfile(file): | |
351 if not os.path.exists(file): | |
352 return '' | |
353 hasher = util.sha1('') | |
354 fd = open(file, 'rb') | |
355 for data in blockstream(fd): | |
356 hasher.update(data) | |
357 fd.close() | |
358 return hasher.hexdigest() | |
359 | |
360 class limitreader(object): | |
361 def __init__(self, f, limit): | |
362 self.f = f | |
363 self.limit = limit | |
364 | |
365 def read(self, length): | |
366 if self.limit == 0: | |
367 return '' | |
368 length = length > self.limit and self.limit or length | |
369 self.limit -= length | |
370 return self.f.read(length) | |
371 | |
372 def close(self): | |
373 pass | |
374 | |
375 def blockstream(infile, blocksize=128 * 1024): | |
376 """Generator that yields blocks of data from infile and closes infile.""" | |
377 while True: | |
378 data = infile.read(blocksize) | |
379 if not data: | |
380 break | |
381 yield data | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
382 # same blecch as copyandhash() above |
15168 | 383 infile.close() |
384 | |
385 def readhash(filename): | |
386 rfile = open(filename, 'rb') | |
387 hash = rfile.read(40) | |
388 rfile.close() | |
389 if len(hash) < 40: | |
390 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') | |
391 % (filename, len(hash))) | |
392 return hash | |
393 | |
394 def writehash(hash, filename, executable): | |
395 util.makedirs(os.path.dirname(filename)) | |
396 if os.path.exists(filename): | |
397 os.unlink(filename) | |
398 wfile = open(filename, 'wb') | |
399 | |
400 try: | |
401 wfile.write(hash) | |
402 wfile.write('\n') | |
403 finally: | |
404 wfile.close() | |
405 if os.path.exists(filename): | |
406 os.chmod(filename, getmode(executable)) | |
407 | |
408 def getexecutable(filename): | |
409 mode = os.stat(filename).st_mode | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
410 return ((mode & stat.S_IXUSR) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
411 (mode & stat.S_IXGRP) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
412 (mode & stat.S_IXOTH)) |
15168 | 413 |
414 def getmode(executable): | |
415 if executable: | |
416 return 0755 | |
417 else: | |
418 return 0644 | |
419 | |
420 def urljoin(first, second, *arg): | |
421 def join(left, right): | |
422 if not left.endswith('/'): | |
423 left += '/' | |
424 if right.startswith('/'): | |
425 right = right[1:] | |
426 return left + right | |
427 | |
428 url = join(first, second) | |
429 for a in arg: | |
430 url = join(url, a) | |
431 return url | |
432 | |
433 def hexsha1(data): | |
434 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | |
435 object data""" | |
15347
799e56609ef6
largefiles: use util.sha1() instead of hashlib.sha1() everywhere
Thomas Arendsen Hein <thomas@intevation.de>
parents:
15333
diff
changeset
|
436 h = util.sha1() |
15168 | 437 for chunk in util.filechunkiter(data): |
438 h.update(chunk) | |
439 return h.hexdigest() | |
440 | |
441 def httpsendfile(ui, filename): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
442 return httpconnection.httpsendfile(ui, filename, 'rb') |
15168 | 443 |
444 def unixpath(path): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
445 '''Return a version of path normalized for use with the lfdirstate.''' |
15168 | 446 return os.path.normpath(path).replace(os.sep, '/') |
447 | |
448 def islfilesrepo(repo): | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
449 return ('largefiles' in repo.requirements and |
15319
9da7e96cd5c2
largefiles: remove redundant any_ function
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15317
diff
changeset
|
450 util.any(shortname + '/' in f[0] for f in repo.store.datafiles())) |
15168 | 451 |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
452 def mkstemp(repo, prefix): |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
453 '''Returns a file descriptor and a filename corresponding to a temporary |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
454 file in the repo's largefiles store.''' |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
455 path = repo.join(longname) |
15392
d7bfbc92a1c0
util: add a doctest for empty sha() calls
Matt Mackall <mpm@selenic.com>
parents:
15391
diff
changeset
|
456 util.makedirs(path) |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
457 return tempfile.mkstemp(prefix=prefix, dir=path) |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
458 |
15333
f37b71fec602
largefiles: py2.4 doesn't have BaseException
Matt Mackall <mpm@selenic.com>
parents:
15320
diff
changeset
|
459 class storeprotonotcapable(Exception): |
15168 | 460 def __init__(self, storetypes): |
461 self.storetypes = storetypes |