Mercurial > hg
annotate hgext/largefiles/lfutil.py @ 15317:41f371150ccb stable
largefiles: make the store primary, and the user cache secondary
This uses the now-properly-named functions and methods from the previous
patch to actually deliver the desired behavior
author | Benjamin Pollack <benjamin@bitquabit.com> |
---|---|
date | Thu, 20 Oct 2011 13:24:11 -0400 |
parents | c65f5b6e26d4 |
children | 9da7e96cd5c2 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''largefiles utility code: must not import other modules in this package.''' | |
10 | |
11 import os | |
12 import errno | |
13 import shutil | |
14 import stat | |
15 import hashlib | |
16 | |
15226
2223ea21c98f
largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
17 from mercurial import dirstate, httpconnection, match as match_, util, scmutil |
15168 | 18 from mercurial.i18n import _ |
19 | |
20 shortname = '.hglf' | |
21 longname = 'largefiles' | |
22 | |
23 | |
24 # -- Portability wrappers ---------------------------------------------- | |
25 | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
26 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
27 return dirstate.walk(matcher, [], unknown, ignored) |
15168 | 28 |
29 def repo_add(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
30 add = repo[None].add |
15168 | 31 return add(list) |
32 | |
33 def repo_remove(repo, list, unlink=False): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
34 def remove(list, unlink): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
35 wlock = repo.wlock() |
15168 | 36 try: |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
37 if unlink: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
38 for f in list: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
39 try: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
40 util.unlinkpath(repo.wjoin(f)) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
41 except OSError, inst: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
42 if inst.errno != errno.ENOENT: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
43 raise |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
44 repo[None].forget(list) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
45 finally: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
46 wlock.release() |
15168 | 47 return remove(list, unlink=unlink) |
48 | |
49 def repo_forget(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
50 forget = repo[None].forget |
15168 | 51 return forget(list) |
52 | |
53 def findoutgoing(repo, remote, force): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
54 from mercurial import discovery |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
55 common, _anyinc, _heads = discovery.findcommonincoming(repo, |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
56 remote, force=force) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
57 return repo.changelog.findmissing(common) |
15168 | 58 |
59 # -- Private worker functions ------------------------------------------ | |
60 | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
61 def getminsize(ui, assumelfiles, opt, default=10): |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
62 lfsize = opt |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
63 if not lfsize and assumelfiles: |
15304
9aa9d4bb3d88
largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents:
15255
diff
changeset
|
64 lfsize = ui.config(longname, 'minsize', default=default) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
65 if lfsize: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
66 try: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
67 lfsize = float(lfsize) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
68 except ValueError: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
69 raise util.Abort(_('largefiles: size must be number (not %s)\n') |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
70 % lfsize) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
71 if lfsize is None: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
72 raise util.Abort(_('minimum size for largefiles must be specified')) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
73 return lfsize |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
74 |
15168 | 75 def link(src, dest): |
76 try: | |
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
77 util.oslink(src, dest) |
15168 | 78 except OSError: |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
79 # if hardlinks fail, fallback on copy |
15168 | 80 shutil.copyfile(src, dest) |
81 os.chmod(dest, os.stat(src).st_mode) | |
82 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
83 def usercachepath(ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
84 path = ui.config(longname, 'usercache', None) |
15168 | 85 if path: |
86 path = os.path.join(path, hash) | |
87 else: | |
88 if os.name == 'nt': | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
89 appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
90 path = os.path.join(appdata, longname, hash) |
15168 | 91 elif os.name == 'posix': |
92 path = os.path.join(os.getenv('HOME'), '.' + longname, hash) | |
93 else: | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
94 raise util.Abort(_('unknown operating system: %s\n') % os.name) |
15168 | 95 return path |
96 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
97 def inusercache(ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
98 return os.path.exists(usercachepath(ui, hash)) |
15168 | 99 |
100 def findfile(repo, hash): | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
101 if instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
102 repo.ui.note(_('Found %s in store\n') % hash) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
103 elif inusercache(repo.ui, hash): |
15168 | 104 repo.ui.note(_('Found %s in system cache\n') % hash) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
105 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
106 else: |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
107 return None |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
108 return storepath(repo, hash) |
15168 | 109 |
110 class largefiles_dirstate(dirstate.dirstate): | |
111 def __getitem__(self, key): | |
112 return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | |
113 def normal(self, f): | |
114 return super(largefiles_dirstate, self).normal(unixpath(f)) | |
115 def remove(self, f): | |
116 return super(largefiles_dirstate, self).remove(unixpath(f)) | |
117 def add(self, f): | |
118 return super(largefiles_dirstate, self).add(unixpath(f)) | |
119 def drop(self, f): | |
120 return super(largefiles_dirstate, self).drop(unixpath(f)) | |
121 def forget(self, f): | |
122 return super(largefiles_dirstate, self).forget(unixpath(f)) | |
123 | |
124 def openlfdirstate(ui, repo): | |
125 ''' | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
126 Return a dirstate object that tracks largefiles: i.e. its root is |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
127 the repo root, but it is saved in .hg/largefiles/dirstate. |
15168 | 128 ''' |
129 admin = repo.join(longname) | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
130 opener = scmutil.opener(admin) |
15169
aa262fff87ac
largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
131 if util.safehasattr(repo.dirstate, '_validate'): |
15168 | 132 lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
133 repo.dirstate._validate) | |
134 else: | |
135 lfdirstate = largefiles_dirstate(opener, ui, repo.root) | |
136 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
137 # If the largefiles dirstate does not exist, populate and create |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
138 # it. This ensures that we create it on the first meaningful |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
139 # largefiles operation in a new clone. It also gives us an easy |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
140 # way to forcibly rebuild largefiles state: |
15168 | 141 # rm .hg/largefiles/dirstate && hg status |
142 # Or even, if things are really messed up: | |
143 # rm -rf .hg/largefiles && hg status | |
144 if not os.path.exists(os.path.join(admin, 'dirstate')): | |
145 util.makedirs(admin) | |
146 matcher = getstandinmatcher(repo) | |
147 for standin in dirstate_walk(repo.dirstate, matcher): | |
148 lfile = splitstandin(standin) | |
149 hash = readstandin(repo, lfile) | |
150 lfdirstate.normallookup(lfile) | |
151 try: | |
152 if hash == hashfile(lfile): | |
153 lfdirstate.normal(lfile) | |
154 except IOError, err: | |
155 if err.errno != errno.ENOENT: | |
156 raise | |
157 | |
158 lfdirstate.write() | |
159 | |
160 return lfdirstate | |
161 | |
162 def lfdirstate_status(lfdirstate, repo, rev): | |
163 wlock = repo.wlock() | |
164 try: | |
165 match = match_.always(repo.root, repo.getcwd()) | |
166 s = lfdirstate.status(match, [], False, False, False) | |
167 unsure, modified, added, removed, missing, unknown, ignored, clean = s | |
168 for lfile in unsure: | |
169 if repo[rev][standin(lfile)].data().strip() != \ | |
170 hashfile(repo.wjoin(lfile)): | |
171 modified.append(lfile) | |
172 else: | |
173 clean.append(lfile) | |
174 lfdirstate.normal(lfile) | |
175 lfdirstate.write() | |
176 finally: | |
177 wlock.release() | |
178 return (modified, added, removed, missing, unknown, ignored, clean) | |
179 | |
180 def listlfiles(repo, rev=None, matcher=None): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
181 '''return a list of largefiles in the working copy or the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
182 specified changeset''' |
15168 | 183 |
184 if matcher is None: | |
185 matcher = getstandinmatcher(repo) | |
186 | |
187 # ignore unknown files in working directory | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
188 return [splitstandin(f) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
189 for f in repo[rev].walk(matcher) |
15168 | 190 if rev is not None or repo.dirstate[f] != '?'] |
191 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
192 def instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
193 return os.path.exists(storepath(repo, hash)) |
15168 | 194 |
195 def createdir(dir): | |
196 if not os.path.exists(dir): | |
197 os.makedirs(dir) | |
198 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
199 def storepath(repo, hash): |
15168 | 200 return repo.join(os.path.join(longname, hash)) |
201 | |
202 def copyfromcache(repo, hash, filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
203 '''Copy the specified largefile from the repo or system cache to |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
204 filename in the repository. Return true on success or false if the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
205 file was not found in either cache (which should not happened: |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
206 this is meant to be called only after ensuring that the needed |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
207 largefile exists in the cache).''' |
15168 | 208 path = findfile(repo, hash) |
209 if path is None: | |
210 return False | |
211 util.makedirs(os.path.dirname(repo.wjoin(filename))) | |
212 shutil.copy(path, repo.wjoin(filename)) | |
213 return True | |
214 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
215 def copytostore(repo, rev, file, uploaded=False): |
15168 | 216 hash = readstandin(repo, file) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
217 if instore(repo, hash): |
15168 | 218 return |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
219 copytostoreabsolute(repo, repo.wjoin(file), hash) |
15168 | 220 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
221 def copytostoreabsolute(repo, file, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
222 createdir(os.path.dirname(storepath(repo, hash))) |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
223 if inusercache(repo.ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
224 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
15168 | 225 else: |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
226 shutil.copyfile(file, storepath(repo, hash)) |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
227 os.chmod(storepath(repo, hash), os.stat(file).st_mode) |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
228 linktousercache(repo, hash) |
15168 | 229 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
230 def linktousercache(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
231 createdir(os.path.dirname(usercachepath(repo.ui, hash))) |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
232 link(storepath(repo, hash), usercachepath(repo.ui, hash)) |
15168 | 233 |
234 def getstandinmatcher(repo, pats=[], opts={}): | |
235 '''Return a match object that applies pats to the standin directory''' | |
236 standindir = repo.pathto(shortname) | |
237 if pats: | |
238 # patterns supplied: search standin directory relative to current dir | |
239 cwd = repo.getcwd() | |
240 if os.path.isabs(cwd): | |
241 # cwd is an absolute path for hg -R <reponame> | |
242 # work relative to the repository root in this case | |
243 cwd = '' | |
244 pats = [os.path.join(standindir, cwd, pat) for pat in pats] | |
245 elif os.path.isdir(standindir): | |
246 # no patterns: relative to repo root | |
247 pats = [standindir] | |
248 else: | |
249 # no patterns and no standin dir: return matcher that matches nothing | |
250 match = match_.match(repo.root, None, [], exact=True) | |
251 match.matchfn = lambda f: False | |
252 return match | |
253 return getmatcher(repo, pats, opts, showbad=False) | |
254 | |
255 def getmatcher(repo, pats=[], opts={}, showbad=True): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
256 '''Wrapper around scmutil.match() that adds showbad: if false, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
257 neuter the match object's bad() method so it does not print any |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
258 warnings about missing files or directories.''' |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
259 match = scmutil.match(repo[None], pats, opts) |
15168 | 260 |
261 if not showbad: | |
262 match.bad = lambda f, msg: None | |
263 return match | |
264 | |
265 def composestandinmatcher(repo, rmatcher): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
266 '''Return a matcher that accepts standins corresponding to the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
267 files accepted by rmatcher. Pass the list of files in the matcher |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
268 as the paths specified by the user.''' |
15168 | 269 smatcher = getstandinmatcher(repo, rmatcher.files()) |
270 isstandin = smatcher.matchfn | |
271 def composed_matchfn(f): | |
272 return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | |
273 smatcher.matchfn = composed_matchfn | |
274 | |
275 return smatcher | |
276 | |
277 def standin(filename): | |
278 '''Return the repo-relative path to the standin for the specified big | |
279 file.''' | |
280 # Notes: | |
281 # 1) Most callers want an absolute path, but _create_standin() needs | |
282 # it repo-relative so lfadd() can pass it to repo_add(). So leave | |
283 # it up to the caller to use repo.wjoin() to get an absolute path. | |
284 # 2) Join with '/' because that's what dirstate always uses, even on | |
285 # Windows. Change existing separator to '/' first in case we are | |
286 # passed filenames from an external source (like the command line). | |
287 return shortname + '/' + filename.replace(os.sep, '/') | |
288 | |
289 def isstandin(filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
290 '''Return true if filename is a big file standin. filename must be |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
291 in Mercurial's internal form (slash-separated).''' |
15168 | 292 return filename.startswith(shortname + '/') |
293 | |
294 def splitstandin(filename): | |
295 # Split on / because that's what dirstate always uses, even on Windows. | |
296 # Change local separator to / first just in case we are passed filenames | |
297 # from an external source (like the command line). | |
298 bits = filename.replace(os.sep, '/').split('/', 1) | |
299 if len(bits) == 2 and bits[0] == shortname: | |
300 return bits[1] | |
301 else: | |
302 return None | |
303 | |
304 def updatestandin(repo, standin): | |
305 file = repo.wjoin(splitstandin(standin)) | |
306 if os.path.exists(file): | |
307 hash = hashfile(file) | |
308 executable = getexecutable(file) | |
309 writestandin(repo, standin, hash, executable) | |
310 | |
311 def readstandin(repo, filename, node=None): | |
312 '''read hex hash from standin for filename at given node, or working | |
313 directory if no node is given''' | |
314 return repo[node][standin(filename)].data().strip() | |
315 | |
316 def writestandin(repo, standin, hash, executable): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
317 '''write hash to <repo.root>/<standin>''' |
15168 | 318 writehash(hash, repo.wjoin(standin), executable) |
319 | |
320 def copyandhash(instream, outfile): | |
321 '''Read bytes from instream (iterable) and write them to outfile, | |
322 computing the SHA-1 hash of the data along the way. Close outfile | |
323 when done and return the binary hash.''' | |
324 hasher = util.sha1('') | |
325 for data in instream: | |
326 hasher.update(data) | |
327 outfile.write(data) | |
328 | |
329 # Blecch: closing a file that somebody else opened is rude and | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
330 # wrong. But it's so darn convenient and practical! After all, |
15168 | 331 # outfile was opened just to copy and hash. |
332 outfile.close() | |
333 | |
334 return hasher.digest() | |
335 | |
336 def hashrepofile(repo, file): | |
337 return hashfile(repo.wjoin(file)) | |
338 | |
339 def hashfile(file): | |
340 if not os.path.exists(file): | |
341 return '' | |
342 hasher = util.sha1('') | |
343 fd = open(file, 'rb') | |
344 for data in blockstream(fd): | |
345 hasher.update(data) | |
346 fd.close() | |
347 return hasher.hexdigest() | |
348 | |
349 class limitreader(object): | |
350 def __init__(self, f, limit): | |
351 self.f = f | |
352 self.limit = limit | |
353 | |
354 def read(self, length): | |
355 if self.limit == 0: | |
356 return '' | |
357 length = length > self.limit and self.limit or length | |
358 self.limit -= length | |
359 return self.f.read(length) | |
360 | |
361 def close(self): | |
362 pass | |
363 | |
364 def blockstream(infile, blocksize=128 * 1024): | |
365 """Generator that yields blocks of data from infile and closes infile.""" | |
366 while True: | |
367 data = infile.read(blocksize) | |
368 if not data: | |
369 break | |
370 yield data | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
371 # same blecch as copyandhash() above |
15168 | 372 infile.close() |
373 | |
374 def readhash(filename): | |
375 rfile = open(filename, 'rb') | |
376 hash = rfile.read(40) | |
377 rfile.close() | |
378 if len(hash) < 40: | |
379 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') | |
380 % (filename, len(hash))) | |
381 return hash | |
382 | |
383 def writehash(hash, filename, executable): | |
384 util.makedirs(os.path.dirname(filename)) | |
385 if os.path.exists(filename): | |
386 os.unlink(filename) | |
387 wfile = open(filename, 'wb') | |
388 | |
389 try: | |
390 wfile.write(hash) | |
391 wfile.write('\n') | |
392 finally: | |
393 wfile.close() | |
394 if os.path.exists(filename): | |
395 os.chmod(filename, getmode(executable)) | |
396 | |
397 def getexecutable(filename): | |
398 mode = os.stat(filename).st_mode | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
399 return ((mode & stat.S_IXUSR) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
400 (mode & stat.S_IXGRP) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
401 (mode & stat.S_IXOTH)) |
15168 | 402 |
403 def getmode(executable): | |
404 if executable: | |
405 return 0755 | |
406 else: | |
407 return 0644 | |
408 | |
409 def urljoin(first, second, *arg): | |
410 def join(left, right): | |
411 if not left.endswith('/'): | |
412 left += '/' | |
413 if right.startswith('/'): | |
414 right = right[1:] | |
415 return left + right | |
416 | |
417 url = join(first, second) | |
418 for a in arg: | |
419 url = join(url, a) | |
420 return url | |
421 | |
422 def hexsha1(data): | |
423 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | |
424 object data""" | |
425 h = hashlib.sha1() | |
426 for chunk in util.filechunkiter(data): | |
427 h.update(chunk) | |
428 return h.hexdigest() | |
429 | |
430 def httpsendfile(ui, filename): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
431 return httpconnection.httpsendfile(ui, filename, 'rb') |
15168 | 432 |
433 def unixpath(path): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
434 '''Return a version of path normalized for use with the lfdirstate.''' |
15168 | 435 return os.path.normpath(path).replace(os.sep, '/') |
436 | |
437 def islfilesrepo(repo): | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
438 return ('largefiles' in repo.requirements and |
15188
8e115063950d
largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents:
15171
diff
changeset
|
439 any_(shortname + '/' in f[0] for f in repo.store.datafiles())) |
15168 | 440 |
441 def any_(gen): | |
442 for x in gen: | |
443 if x: | |
444 return True | |
445 return False | |
446 | |
447 class storeprotonotcapable(BaseException): | |
448 def __init__(self, storetypes): | |
449 self.storetypes = storetypes |