Mercurial > hg
annotate hgext/largefiles/lfutil.py @ 15793:3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
When rebasing, we need to trust that the standins are always correct. The
rebase operation updates the standins according to the changeset it is
rebasing. We need to make the largefiles in the working copy match. If we
don't make them match, then they get accidentally reverted, either during
the rebase or during the next commit after the rebase.
This worked previously only becuase we were relying on the behavior that
largefiles with a changed standin, but unchanged contents, never showed up in
the list of modified largefiles. Unfortunately, pre-commit hooks can get
an incorrect status this way, and it also results in extra execution of code.
The solution is to simply trust the standins when we are about to commit a
rebased changeset, and politely ask updatelfiles() to pull the new contents
down. In this case, updatelfiles() will also mark any files it has pulled
down as dirty in the lfdirstate so that pre-commit hooks will get correct
status output.
author | Na'Tosha Bard <natosha@unity3d.com> |
---|---|
date | Sat, 07 Jan 2012 18:43:34 +0100 |
parents | 1facaad963a8 |
children | 0d91211dd12f |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''largefiles utility code: must not import other modules in this package.''' | |
10 | |
11 import os | |
12 import errno | |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
13 import platform |
15168 | 14 import shutil |
15 import stat | |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
16 import tempfile |
15168 | 17 |
15226
2223ea21c98f
largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
18 from mercurial import dirstate, httpconnection, match as match_, util, scmutil |
15168 | 19 from mercurial.i18n import _ |
20 | |
21 shortname = '.hglf' | |
22 longname = 'largefiles' | |
23 | |
24 | |
25 # -- Portability wrappers ---------------------------------------------- | |
26 | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
27 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
28 return dirstate.walk(matcher, [], unknown, ignored) |
15168 | 29 |
30 def repo_add(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
31 add = repo[None].add |
15168 | 32 return add(list) |
33 | |
34 def repo_remove(repo, list, unlink=False): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
35 def remove(list, unlink): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
36 wlock = repo.wlock() |
15168 | 37 try: |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
38 if unlink: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
39 for f in list: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
40 try: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
41 util.unlinkpath(repo.wjoin(f)) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
42 except OSError, inst: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
43 if inst.errno != errno.ENOENT: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
44 raise |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
45 repo[None].forget(list) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
46 finally: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
47 wlock.release() |
15168 | 48 return remove(list, unlink=unlink) |
49 | |
50 def repo_forget(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
51 forget = repo[None].forget |
15168 | 52 return forget(list) |
53 | |
54 def findoutgoing(repo, remote, force): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
55 from mercurial import discovery |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
56 common, _anyinc, _heads = discovery.findcommonincoming(repo, |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
57 remote, force=force) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
58 return repo.changelog.findmissing(common) |
15168 | 59 |
60 # -- Private worker functions ------------------------------------------ | |
61 | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
62 def getminsize(ui, assumelfiles, opt, default=10): |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
63 lfsize = opt |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
64 if not lfsize and assumelfiles: |
15304
9aa9d4bb3d88
largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents:
15255
diff
changeset
|
65 lfsize = ui.config(longname, 'minsize', default=default) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
66 if lfsize: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
67 try: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
68 lfsize = float(lfsize) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
69 except ValueError: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
70 raise util.Abort(_('largefiles: size must be number (not %s)\n') |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
71 % lfsize) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
72 if lfsize is None: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
73 raise util.Abort(_('minimum size for largefiles must be specified')) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
74 return lfsize |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
75 |
15168 | 76 def link(src, dest): |
77 try: | |
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
78 util.oslink(src, dest) |
15168 | 79 except OSError: |
15572
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
80 # if hardlinks fail, fallback on atomic copy |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
81 dst = util.atomictempfile(dest) |
15699
84e55467093c
largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents:
15658
diff
changeset
|
82 for chunk in util.filechunkiter(open(src, 'rb')): |
15572
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
83 dst.write(chunk) |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
84 dst.close() |
15168 | 85 os.chmod(dest, os.stat(src).st_mode) |
86 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
87 def usercachepath(ui, hash): |
15350
8b8dd13295db
largefiles: use ui.configpath() where appropriate
Greg Ward <greg@gerg.ca>
parents:
15349
diff
changeset
|
88 path = ui.configpath(longname, 'usercache', None) |
15168 | 89 if path: |
90 path = os.path.join(path, hash) | |
91 else: | |
92 if os.name == 'nt': | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
93 appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
94 if appdata: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
95 path = os.path.join(appdata, longname, hash) |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
96 elif platform.system() == 'Darwin': |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
97 home = os.getenv('HOME') |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
98 if home: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
99 path = os.path.join(home, 'Library', 'Caches', |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
100 longname, hash) |
15168 | 101 elif os.name == 'posix': |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
102 path = os.getenv('XDG_CACHE_HOME') |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
103 if path: |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
104 path = os.path.join(path, longname, hash) |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
105 else: |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
106 home = os.getenv('HOME') |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
107 if home: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
108 path = os.path.join(home, '.cache', longname, hash) |
15168 | 109 else: |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
110 raise util.Abort(_('unknown operating system: %s\n') % os.name) |
15168 | 111 return path |
112 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
113 def inusercache(ui, hash): |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
114 path = usercachepath(ui, hash) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
115 return path and os.path.exists(path) |
15168 | 116 |
117 def findfile(repo, hash): | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
118 if instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
119 repo.ui.note(_('Found %s in store\n') % hash) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
120 elif inusercache(repo.ui, hash): |
15168 | 121 repo.ui.note(_('Found %s in system cache\n') % hash) |
15408
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
122 path = storepath(repo, hash) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
123 util.makedirs(os.path.dirname(path)) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
124 link(usercachepath(repo.ui, hash), path) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
125 else: |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
126 return None |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
127 return storepath(repo, hash) |
15168 | 128 |
129 class largefiles_dirstate(dirstate.dirstate): | |
130 def __getitem__(self, key): | |
131 return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | |
132 def normal(self, f): | |
133 return super(largefiles_dirstate, self).normal(unixpath(f)) | |
134 def remove(self, f): | |
135 return super(largefiles_dirstate, self).remove(unixpath(f)) | |
136 def add(self, f): | |
137 return super(largefiles_dirstate, self).add(unixpath(f)) | |
138 def drop(self, f): | |
139 return super(largefiles_dirstate, self).drop(unixpath(f)) | |
140 def forget(self, f): | |
141 return super(largefiles_dirstate, self).forget(unixpath(f)) | |
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15700
diff
changeset
|
142 def normallookup(self, f): |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15700
diff
changeset
|
143 return super(largefiles_dirstate, self).normallookup(unixpath(f)) |
15168 | 144 |
145 def openlfdirstate(ui, repo): | |
146 ''' | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
147 Return a dirstate object that tracks largefiles: i.e. its root is |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
148 the repo root, but it is saved in .hg/largefiles/dirstate. |
15168 | 149 ''' |
150 admin = repo.join(longname) | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
151 opener = scmutil.opener(admin) |
15349
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
152 lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
153 repo.dirstate._validate) |
15168 | 154 |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
155 # If the largefiles dirstate does not exist, populate and create |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
156 # it. This ensures that we create it on the first meaningful |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
157 # largefiles operation in a new clone. It also gives us an easy |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
158 # way to forcibly rebuild largefiles state: |
15168 | 159 # rm .hg/largefiles/dirstate && hg status |
160 # Or even, if things are really messed up: | |
161 # rm -rf .hg/largefiles && hg status | |
162 if not os.path.exists(os.path.join(admin, 'dirstate')): | |
163 util.makedirs(admin) | |
164 matcher = getstandinmatcher(repo) | |
165 for standin in dirstate_walk(repo.dirstate, matcher): | |
166 lfile = splitstandin(standin) | |
167 hash = readstandin(repo, lfile) | |
168 lfdirstate.normallookup(lfile) | |
169 try: | |
15553
e89385e4ef8d
largefiles: file storage should be relative to repo, not relative to cwd
Mads Kiilerich <mads@kiilerich.com>
parents:
15548
diff
changeset
|
170 if hash == hashfile(repo.wjoin(lfile)): |
15168 | 171 lfdirstate.normal(lfile) |
15548
f76584098c88
largefiles: fix 'hg clone . ../foo' OSError abort
Martin Geisler <mg@lazybytes.net>
parents:
15408
diff
changeset
|
172 except OSError, err: |
15168 | 173 if err.errno != errno.ENOENT: |
174 raise | |
175 | |
176 lfdirstate.write() | |
177 | |
178 return lfdirstate | |
179 | |
180 def lfdirstate_status(lfdirstate, repo, rev): | |
181 wlock = repo.wlock() | |
182 try: | |
183 match = match_.always(repo.root, repo.getcwd()) | |
184 s = lfdirstate.status(match, [], False, False, False) | |
185 unsure, modified, added, removed, missing, unknown, ignored, clean = s | |
186 for lfile in unsure: | |
187 if repo[rev][standin(lfile)].data().strip() != \ | |
188 hashfile(repo.wjoin(lfile)): | |
189 modified.append(lfile) | |
190 else: | |
191 clean.append(lfile) | |
192 lfdirstate.normal(lfile) | |
193 lfdirstate.write() | |
194 finally: | |
195 wlock.release() | |
196 return (modified, added, removed, missing, unknown, ignored, clean) | |
197 | |
198 def listlfiles(repo, rev=None, matcher=None): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
199 '''return a list of largefiles in the working copy or the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
200 specified changeset''' |
15168 | 201 |
202 if matcher is None: | |
203 matcher = getstandinmatcher(repo) | |
204 | |
205 # ignore unknown files in working directory | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
206 return [splitstandin(f) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
207 for f in repo[rev].walk(matcher) |
15168 | 208 if rev is not None or repo.dirstate[f] != '?'] |
209 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
210 def instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
211 return os.path.exists(storepath(repo, hash)) |
15168 | 212 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
213 def storepath(repo, hash): |
15168 | 214 return repo.join(os.path.join(longname, hash)) |
215 | |
216 def copyfromcache(repo, hash, filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
217 '''Copy the specified largefile from the repo or system cache to |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
218 filename in the repository. Return true on success or false if the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
219 file was not found in either cache (which should not happened: |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
220 this is meant to be called only after ensuring that the needed |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
221 largefile exists in the cache).''' |
15168 | 222 path = findfile(repo, hash) |
223 if path is None: | |
224 return False | |
225 util.makedirs(os.path.dirname(repo.wjoin(filename))) | |
15570
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
226 # The write may fail before the file is fully written, but we |
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
227 # don't use atomic writes in the working copy. |
15168 | 228 shutil.copy(path, repo.wjoin(filename)) |
229 return True | |
230 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
231 def copytostore(repo, rev, file, uploaded=False): |
15168 | 232 hash = readstandin(repo, file) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
233 if instore(repo, hash): |
15168 | 234 return |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
235 copytostoreabsolute(repo, repo.wjoin(file), hash) |
15168 | 236 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
237 def copytostoreabsolute(repo, file, hash): |
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15350
diff
changeset
|
238 util.makedirs(os.path.dirname(storepath(repo, hash))) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
239 if inusercache(repo.ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
240 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
15168 | 241 else: |
15571
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
242 dst = util.atomictempfile(storepath(repo, hash)) |
15699
84e55467093c
largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents:
15658
diff
changeset
|
243 for chunk in util.filechunkiter(open(file, 'rb')): |
15571
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
244 dst.write(chunk) |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
245 dst.close() |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
246 util.copymode(file, storepath(repo, hash)) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
247 linktousercache(repo, hash) |
15168 | 248 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
249 def linktousercache(repo, hash): |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
250 path = usercachepath(repo.ui, hash) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
251 if path: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
252 util.makedirs(os.path.dirname(path)) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
253 link(storepath(repo, hash), path) |
15168 | 254 |
255 def getstandinmatcher(repo, pats=[], opts={}): | |
256 '''Return a match object that applies pats to the standin directory''' | |
257 standindir = repo.pathto(shortname) | |
258 if pats: | |
259 # patterns supplied: search standin directory relative to current dir | |
260 cwd = repo.getcwd() | |
261 if os.path.isabs(cwd): | |
262 # cwd is an absolute path for hg -R <reponame> | |
263 # work relative to the repository root in this case | |
264 cwd = '' | |
265 pats = [os.path.join(standindir, cwd, pat) for pat in pats] | |
266 elif os.path.isdir(standindir): | |
267 # no patterns: relative to repo root | |
268 pats = [standindir] | |
269 else: | |
270 # no patterns and no standin dir: return matcher that matches nothing | |
271 match = match_.match(repo.root, None, [], exact=True) | |
272 match.matchfn = lambda f: False | |
273 return match | |
274 return getmatcher(repo, pats, opts, showbad=False) | |
275 | |
276 def getmatcher(repo, pats=[], opts={}, showbad=True): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
277 '''Wrapper around scmutil.match() that adds showbad: if false, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
278 neuter the match object's bad() method so it does not print any |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
279 warnings about missing files or directories.''' |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
280 match = scmutil.match(repo[None], pats, opts) |
15168 | 281 |
282 if not showbad: | |
283 match.bad = lambda f, msg: None | |
284 return match | |
285 | |
286 def composestandinmatcher(repo, rmatcher): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
287 '''Return a matcher that accepts standins corresponding to the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
288 files accepted by rmatcher. Pass the list of files in the matcher |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
289 as the paths specified by the user.''' |
15168 | 290 smatcher = getstandinmatcher(repo, rmatcher.files()) |
291 isstandin = smatcher.matchfn | |
292 def composed_matchfn(f): | |
293 return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | |
294 smatcher.matchfn = composed_matchfn | |
295 | |
296 return smatcher | |
297 | |
298 def standin(filename): | |
299 '''Return the repo-relative path to the standin for the specified big | |
300 file.''' | |
301 # Notes: | |
302 # 1) Most callers want an absolute path, but _create_standin() needs | |
303 # it repo-relative so lfadd() can pass it to repo_add(). So leave | |
304 # it up to the caller to use repo.wjoin() to get an absolute path. | |
305 # 2) Join with '/' because that's what dirstate always uses, even on | |
306 # Windows. Change existing separator to '/' first in case we are | |
307 # passed filenames from an external source (like the command line). | |
308 return shortname + '/' + filename.replace(os.sep, '/') | |
309 | |
310 def isstandin(filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
311 '''Return true if filename is a big file standin. filename must be |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
312 in Mercurial's internal form (slash-separated).''' |
15168 | 313 return filename.startswith(shortname + '/') |
314 | |
315 def splitstandin(filename): | |
316 # Split on / because that's what dirstate always uses, even on Windows. | |
317 # Change local separator to / first just in case we are passed filenames | |
318 # from an external source (like the command line). | |
319 bits = filename.replace(os.sep, '/').split('/', 1) | |
320 if len(bits) == 2 and bits[0] == shortname: | |
321 return bits[1] | |
322 else: | |
323 return None | |
324 | |
325 def updatestandin(repo, standin): | |
326 file = repo.wjoin(splitstandin(standin)) | |
327 if os.path.exists(file): | |
328 hash = hashfile(file) | |
329 executable = getexecutable(file) | |
330 writestandin(repo, standin, hash, executable) | |
331 | |
332 def readstandin(repo, filename, node=None): | |
333 '''read hex hash from standin for filename at given node, or working | |
334 directory if no node is given''' | |
335 return repo[node][standin(filename)].data().strip() | |
336 | |
337 def writestandin(repo, standin, hash, executable): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
338 '''write hash to <repo.root>/<standin>''' |
15168 | 339 writehash(hash, repo.wjoin(standin), executable) |
340 | |
341 def copyandhash(instream, outfile): | |
342 '''Read bytes from instream (iterable) and write them to outfile, | |
343 computing the SHA-1 hash of the data along the way. Close outfile | |
344 when done and return the binary hash.''' | |
345 hasher = util.sha1('') | |
346 for data in instream: | |
347 hasher.update(data) | |
348 outfile.write(data) | |
349 | |
350 # Blecch: closing a file that somebody else opened is rude and | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
351 # wrong. But it's so darn convenient and practical! After all, |
15168 | 352 # outfile was opened just to copy and hash. |
353 outfile.close() | |
354 | |
355 return hasher.digest() | |
356 | |
357 def hashrepofile(repo, file): | |
358 return hashfile(repo.wjoin(file)) | |
359 | |
360 def hashfile(file): | |
361 if not os.path.exists(file): | |
362 return '' | |
363 hasher = util.sha1('') | |
364 fd = open(file, 'rb') | |
365 for data in blockstream(fd): | |
366 hasher.update(data) | |
367 fd.close() | |
368 return hasher.hexdigest() | |
369 | |
370 class limitreader(object): | |
371 def __init__(self, f, limit): | |
372 self.f = f | |
373 self.limit = limit | |
374 | |
375 def read(self, length): | |
376 if self.limit == 0: | |
377 return '' | |
378 length = length > self.limit and self.limit or length | |
379 self.limit -= length | |
380 return self.f.read(length) | |
381 | |
382 def close(self): | |
383 pass | |
384 | |
385 def blockstream(infile, blocksize=128 * 1024): | |
386 """Generator that yields blocks of data from infile and closes infile.""" | |
387 while True: | |
388 data = infile.read(blocksize) | |
389 if not data: | |
390 break | |
391 yield data | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
392 # same blecch as copyandhash() above |
15168 | 393 infile.close() |
394 | |
395 def readhash(filename): | |
396 rfile = open(filename, 'rb') | |
397 hash = rfile.read(40) | |
398 rfile.close() | |
399 if len(hash) < 40: | |
400 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') | |
401 % (filename, len(hash))) | |
402 return hash | |
403 | |
404 def writehash(hash, filename, executable): | |
405 util.makedirs(os.path.dirname(filename)) | |
15574
c9328c829cd9
largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents:
15572
diff
changeset
|
406 util.writefile(filename, hash + '\n') |
c9328c829cd9
largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents:
15572
diff
changeset
|
407 os.chmod(filename, getmode(executable)) |
15168 | 408 |
409 def getexecutable(filename): | |
410 mode = os.stat(filename).st_mode | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
411 return ((mode & stat.S_IXUSR) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
412 (mode & stat.S_IXGRP) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
413 (mode & stat.S_IXOTH)) |
15168 | 414 |
415 def getmode(executable): | |
416 if executable: | |
417 return 0755 | |
418 else: | |
419 return 0644 | |
420 | |
421 def urljoin(first, second, *arg): | |
422 def join(left, right): | |
423 if not left.endswith('/'): | |
424 left += '/' | |
425 if right.startswith('/'): | |
426 right = right[1:] | |
427 return left + right | |
428 | |
429 url = join(first, second) | |
430 for a in arg: | |
431 url = join(url, a) | |
432 return url | |
433 | |
434 def hexsha1(data): | |
435 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | |
436 object data""" | |
15347
799e56609ef6
largefiles: use util.sha1() instead of hashlib.sha1() everywhere
Thomas Arendsen Hein <thomas@intevation.de>
parents:
15333
diff
changeset
|
437 h = util.sha1() |
15168 | 438 for chunk in util.filechunkiter(data): |
439 h.update(chunk) | |
440 return h.hexdigest() | |
441 | |
442 def httpsendfile(ui, filename): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
443 return httpconnection.httpsendfile(ui, filename, 'rb') |
15168 | 444 |
445 def unixpath(path): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
446 '''Return a version of path normalized for use with the lfdirstate.''' |
15168 | 447 return os.path.normpath(path).replace(os.sep, '/') |
448 | |
449 def islfilesrepo(repo): | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
450 return ('largefiles' in repo.requirements and |
15319
9da7e96cd5c2
largefiles: remove redundant any_ function
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15317
diff
changeset
|
451 util.any(shortname + '/' in f[0] for f in repo.store.datafiles())) |
15168 | 452 |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
453 def mkstemp(repo, prefix): |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
454 '''Returns a file descriptor and a filename corresponding to a temporary |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
455 file in the repo's largefiles store.''' |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
456 path = repo.join(longname) |
15392
d7bfbc92a1c0
util: add a doctest for empty sha() calls
Matt Mackall <mpm@selenic.com>
parents:
15391
diff
changeset
|
457 util.makedirs(path) |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
458 return tempfile.mkstemp(prefix=prefix, dir=path) |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
459 |
15333
f37b71fec602
largefiles: py2.4 doesn't have BaseException
Matt Mackall <mpm@selenic.com>
parents:
15320
diff
changeset
|
460 class storeprotonotcapable(Exception): |
15168 | 461 def __init__(self, storetypes): |
462 self.storetypes = storetypes |