Mercurial > hg-stable
annotate hgext/largefiles/lfutil.py @ 15794:0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Don't lock/write on operations that should be readonly (status).
Always lock when writing the lfdirstate (rollback).
Don't write lfdirstate until after committing; state isn't actually changed
until the commit is complete.
author | Levi Bard <levi@unity3d.com> |
---|---|
date | Sat, 07 Jan 2012 19:05:59 +0100 |
parents | 3ef07ecdb0d5 |
children | 3e5b6045ccfc |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''largefiles utility code: must not import other modules in this package.''' | |
10 | |
11 import os | |
12 import errno | |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
13 import platform |
15168 | 14 import shutil |
15 import stat | |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
16 import tempfile |
15168 | 17 |
15226
2223ea21c98f
largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
18 from mercurial import dirstate, httpconnection, match as match_, util, scmutil |
15168 | 19 from mercurial.i18n import _ |
20 | |
21 shortname = '.hglf' | |
22 longname = 'largefiles' | |
23 | |
24 | |
25 # -- Portability wrappers ---------------------------------------------- | |
26 | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
27 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
28 return dirstate.walk(matcher, [], unknown, ignored) |
15168 | 29 |
30 def repo_add(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
31 add = repo[None].add |
15168 | 32 return add(list) |
33 | |
34 def repo_remove(repo, list, unlink=False): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
35 def remove(list, unlink): |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
36 wlock = repo.wlock() |
15168 | 37 try: |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
38 if unlink: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
39 for f in list: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
40 try: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
41 util.unlinkpath(repo.wjoin(f)) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
42 except OSError, inst: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
43 if inst.errno != errno.ENOENT: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
44 raise |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
45 repo[None].forget(list) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
46 finally: |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
47 wlock.release() |
15168 | 48 return remove(list, unlink=unlink) |
49 | |
50 def repo_forget(repo, list): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
51 forget = repo[None].forget |
15168 | 52 return forget(list) |
53 | |
54 def findoutgoing(repo, remote, force): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
55 from mercurial import discovery |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
56 common, _anyinc, _heads = discovery.findcommonincoming(repo, |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
57 remote, force=force) |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
58 return repo.changelog.findmissing(common) |
15168 | 59 |
60 # -- Private worker functions ------------------------------------------ | |
61 | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
62 def getminsize(ui, assumelfiles, opt, default=10): |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
63 lfsize = opt |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
64 if not lfsize and assumelfiles: |
15304
9aa9d4bb3d88
largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents:
15255
diff
changeset
|
65 lfsize = ui.config(longname, 'minsize', default=default) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
66 if lfsize: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
67 try: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
68 lfsize = float(lfsize) |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
69 except ValueError: |
15228
ee625de3541e
largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents:
15227
diff
changeset
|
70 raise util.Abort(_('largefiles: size must be number (not %s)\n') |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
71 % lfsize) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
72 if lfsize is None: |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
73 raise util.Abort(_('minimum size for largefiles must be specified')) |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
74 return lfsize |
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15226
diff
changeset
|
75 |
15168 | 76 def link(src, dest): |
77 try: | |
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
78 util.oslink(src, dest) |
15168 | 79 except OSError: |
15572
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
80 # if hardlinks fail, fallback on atomic copy |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
81 dst = util.atomictempfile(dest) |
15699
84e55467093c
largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents:
15658
diff
changeset
|
82 for chunk in util.filechunkiter(open(src, 'rb')): |
15572
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
83 dst.write(chunk) |
926bc23d0b6a
largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents:
15571
diff
changeset
|
84 dst.close() |
15168 | 85 os.chmod(dest, os.stat(src).st_mode) |
86 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
87 def usercachepath(ui, hash): |
15350
8b8dd13295db
largefiles: use ui.configpath() where appropriate
Greg Ward <greg@gerg.ca>
parents:
15349
diff
changeset
|
88 path = ui.configpath(longname, 'usercache', None) |
15168 | 89 if path: |
90 path = os.path.join(path, hash) | |
91 else: | |
92 if os.name == 'nt': | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
93 appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
94 if appdata: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
95 path = os.path.join(appdata, longname, hash) |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
96 elif platform.system() == 'Darwin': |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
97 home = os.getenv('HOME') |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
98 if home: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
99 path = os.path.join(home, 'Library', 'Caches', |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
100 longname, hash) |
15168 | 101 elif os.name == 'posix': |
15320
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
102 path = os.getenv('XDG_CACHE_HOME') |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
103 if path: |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
104 path = os.path.join(path, longname, hash) |
681267a5f491
largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15319
diff
changeset
|
105 else: |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
106 home = os.getenv('HOME') |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
107 if home: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
108 path = os.path.join(home, '.cache', longname, hash) |
15168 | 109 else: |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
110 raise util.Abort(_('unknown operating system: %s\n') % os.name) |
15168 | 111 return path |
112 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
113 def inusercache(ui, hash): |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
114 path = usercachepath(ui, hash) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
115 return path and os.path.exists(path) |
15168 | 116 |
117 def findfile(repo, hash): | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
118 if instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
119 repo.ui.note(_('Found %s in store\n') % hash) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
120 elif inusercache(repo.ui, hash): |
15168 | 121 repo.ui.note(_('Found %s in system cache\n') % hash) |
15408
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
122 path = storepath(repo, hash) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
123 util.makedirs(os.path.dirname(path)) |
db8b0ee74025
largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents:
15392
diff
changeset
|
124 link(usercachepath(repo.ui, hash), path) |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
125 else: |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
126 return None |
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15316
diff
changeset
|
127 return storepath(repo, hash) |
15168 | 128 |
129 class largefiles_dirstate(dirstate.dirstate): | |
130 def __getitem__(self, key): | |
131 return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | |
132 def normal(self, f): | |
133 return super(largefiles_dirstate, self).normal(unixpath(f)) | |
134 def remove(self, f): | |
135 return super(largefiles_dirstate, self).remove(unixpath(f)) | |
136 def add(self, f): | |
137 return super(largefiles_dirstate, self).add(unixpath(f)) | |
138 def drop(self, f): | |
139 return super(largefiles_dirstate, self).drop(unixpath(f)) | |
140 def forget(self, f): | |
141 return super(largefiles_dirstate, self).forget(unixpath(f)) | |
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15700
diff
changeset
|
142 def normallookup(self, f): |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15700
diff
changeset
|
143 return super(largefiles_dirstate, self).normallookup(unixpath(f)) |
15168 | 144 |
145 def openlfdirstate(ui, repo): | |
146 ''' | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
147 Return a dirstate object that tracks largefiles: i.e. its root is |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
148 the repo root, but it is saved in .hg/largefiles/dirstate. |
15168 | 149 ''' |
150 admin = repo.join(longname) | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
151 opener = scmutil.opener(admin) |
15349
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
152 lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
63455eb771af
largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents:
15347
diff
changeset
|
153 repo.dirstate._validate) |
15168 | 154 |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
155 # If the largefiles dirstate does not exist, populate and create |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
156 # it. This ensures that we create it on the first meaningful |
15794
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
157 # largefiles operation in a new clone. |
15168 | 158 if not os.path.exists(os.path.join(admin, 'dirstate')): |
159 util.makedirs(admin) | |
160 matcher = getstandinmatcher(repo) | |
161 for standin in dirstate_walk(repo.dirstate, matcher): | |
162 lfile = splitstandin(standin) | |
163 hash = readstandin(repo, lfile) | |
164 lfdirstate.normallookup(lfile) | |
165 try: | |
15553
e89385e4ef8d
largefiles: file storage should be relative to repo, not relative to cwd
Mads Kiilerich <mads@kiilerich.com>
parents:
15548
diff
changeset
|
166 if hash == hashfile(repo.wjoin(lfile)): |
15168 | 167 lfdirstate.normal(lfile) |
15548
f76584098c88
largefiles: fix 'hg clone . ../foo' OSError abort
Martin Geisler <mg@lazybytes.net>
parents:
15408
diff
changeset
|
168 except OSError, err: |
15168 | 169 if err.errno != errno.ENOENT: |
170 raise | |
171 return lfdirstate | |
172 | |
173 def lfdirstate_status(lfdirstate, repo, rev): | |
15794
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
174 match = match_.always(repo.root, repo.getcwd()) |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
175 s = lfdirstate.status(match, [], False, False, False) |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
176 unsure, modified, added, removed, missing, unknown, ignored, clean = s |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
177 for lfile in unsure: |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
178 if repo[rev][standin(lfile)].data().strip() != \ |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
179 hashfile(repo.wjoin(lfile)): |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
180 modified.append(lfile) |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
181 else: |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
182 clean.append(lfile) |
0d91211dd12f
largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
183 lfdirstate.normal(lfile) |
15168 | 184 return (modified, added, removed, missing, unknown, ignored, clean) |
185 | |
186 def listlfiles(repo, rev=None, matcher=None): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
187 '''return a list of largefiles in the working copy or the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
188 specified changeset''' |
15168 | 189 |
190 if matcher is None: | |
191 matcher = getstandinmatcher(repo) | |
192 | |
193 # ignore unknown files in working directory | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
194 return [splitstandin(f) |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
195 for f in repo[rev].walk(matcher) |
15168 | 196 if rev is not None or repo.dirstate[f] != '?'] |
197 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
198 def instore(repo, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
199 return os.path.exists(storepath(repo, hash)) |
15168 | 200 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
201 def storepath(repo, hash): |
15168 | 202 return repo.join(os.path.join(longname, hash)) |
203 | |
204 def copyfromcache(repo, hash, filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
205 '''Copy the specified largefile from the repo or system cache to |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
206 filename in the repository. Return true on success or false if the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
207 file was not found in either cache (which should not happened: |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
208 this is meant to be called only after ensuring that the needed |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
209 largefile exists in the cache).''' |
15168 | 210 path = findfile(repo, hash) |
211 if path is None: | |
212 return False | |
213 util.makedirs(os.path.dirname(repo.wjoin(filename))) | |
15570
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
214 # The write may fail before the file is fully written, but we |
0f208626d503
largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents:
15553
diff
changeset
|
215 # don't use atomic writes in the working copy. |
15168 | 216 shutil.copy(path, repo.wjoin(filename)) |
217 return True | |
218 | |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
219 def copytostore(repo, rev, file, uploaded=False): |
15168 | 220 hash = readstandin(repo, file) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
221 if instore(repo, hash): |
15168 | 222 return |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
223 copytostoreabsolute(repo, repo.wjoin(file), hash) |
15168 | 224 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
225 def copytostoreabsolute(repo, file, hash): |
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15350
diff
changeset
|
226 util.makedirs(os.path.dirname(storepath(repo, hash))) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
227 if inusercache(repo.ui, hash): |
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
228 link(usercachepath(repo.ui, hash), storepath(repo, hash)) |
15168 | 229 else: |
15571
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
230 dst = util.atomictempfile(storepath(repo, hash)) |
15699
84e55467093c
largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents:
15658
diff
changeset
|
231 for chunk in util.filechunkiter(open(file, 'rb')): |
15571
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
232 dst.write(chunk) |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
233 dst.close() |
809788118aa2
largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents:
15570
diff
changeset
|
234 util.copymode(file, storepath(repo, hash)) |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
235 linktousercache(repo, hash) |
15168 | 236 |
15316
c65f5b6e26d4
largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15304
diff
changeset
|
237 def linktousercache(repo, hash): |
15658
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
238 path = usercachepath(repo.ui, hash) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
239 if path: |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
240 util.makedirs(os.path.dirname(path)) |
971c55ce03b8
largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents:
15572
diff
changeset
|
241 link(storepath(repo, hash), path) |
15168 | 242 |
243 def getstandinmatcher(repo, pats=[], opts={}): | |
244 '''Return a match object that applies pats to the standin directory''' | |
245 standindir = repo.pathto(shortname) | |
246 if pats: | |
247 # patterns supplied: search standin directory relative to current dir | |
248 cwd = repo.getcwd() | |
249 if os.path.isabs(cwd): | |
250 # cwd is an absolute path for hg -R <reponame> | |
251 # work relative to the repository root in this case | |
252 cwd = '' | |
253 pats = [os.path.join(standindir, cwd, pat) for pat in pats] | |
254 elif os.path.isdir(standindir): | |
255 # no patterns: relative to repo root | |
256 pats = [standindir] | |
257 else: | |
258 # no patterns and no standin dir: return matcher that matches nothing | |
259 match = match_.match(repo.root, None, [], exact=True) | |
260 match.matchfn = lambda f: False | |
261 return match | |
262 return getmatcher(repo, pats, opts, showbad=False) | |
263 | |
264 def getmatcher(repo, pats=[], opts={}, showbad=True): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
265 '''Wrapper around scmutil.match() that adds showbad: if false, |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
266 neuter the match object's bad() method so it does not print any |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
267 warnings about missing files or directories.''' |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
268 match = scmutil.match(repo[None], pats, opts) |
15168 | 269 |
270 if not showbad: | |
271 match.bad = lambda f, msg: None | |
272 return match | |
273 | |
274 def composestandinmatcher(repo, rmatcher): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
275 '''Return a matcher that accepts standins corresponding to the |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
276 files accepted by rmatcher. Pass the list of files in the matcher |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
277 as the paths specified by the user.''' |
15168 | 278 smatcher = getstandinmatcher(repo, rmatcher.files()) |
279 isstandin = smatcher.matchfn | |
280 def composed_matchfn(f): | |
281 return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | |
282 smatcher.matchfn = composed_matchfn | |
283 | |
284 return smatcher | |
285 | |
286 def standin(filename): | |
287 '''Return the repo-relative path to the standin for the specified big | |
288 file.''' | |
289 # Notes: | |
290 # 1) Most callers want an absolute path, but _create_standin() needs | |
291 # it repo-relative so lfadd() can pass it to repo_add(). So leave | |
292 # it up to the caller to use repo.wjoin() to get an absolute path. | |
293 # 2) Join with '/' because that's what dirstate always uses, even on | |
294 # Windows. Change existing separator to '/' first in case we are | |
295 # passed filenames from an external source (like the command line). | |
296 return shortname + '/' + filename.replace(os.sep, '/') | |
297 | |
298 def isstandin(filename): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
299 '''Return true if filename is a big file standin. filename must be |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
300 in Mercurial's internal form (slash-separated).''' |
15168 | 301 return filename.startswith(shortname + '/') |
302 | |
303 def splitstandin(filename): | |
304 # Split on / because that's what dirstate always uses, even on Windows. | |
305 # Change local separator to / first just in case we are passed filenames | |
306 # from an external source (like the command line). | |
307 bits = filename.replace(os.sep, '/').split('/', 1) | |
308 if len(bits) == 2 and bits[0] == shortname: | |
309 return bits[1] | |
310 else: | |
311 return None | |
312 | |
313 def updatestandin(repo, standin): | |
314 file = repo.wjoin(splitstandin(standin)) | |
315 if os.path.exists(file): | |
316 hash = hashfile(file) | |
317 executable = getexecutable(file) | |
318 writestandin(repo, standin, hash, executable) | |
319 | |
320 def readstandin(repo, filename, node=None): | |
321 '''read hex hash from standin for filename at given node, or working | |
322 directory if no node is given''' | |
323 return repo[node][standin(filename)].data().strip() | |
324 | |
325 def writestandin(repo, standin, hash, executable): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
326 '''write hash to <repo.root>/<standin>''' |
15168 | 327 writehash(hash, repo.wjoin(standin), executable) |
328 | |
329 def copyandhash(instream, outfile): | |
330 '''Read bytes from instream (iterable) and write them to outfile, | |
331 computing the SHA-1 hash of the data along the way. Close outfile | |
332 when done and return the binary hash.''' | |
333 hasher = util.sha1('') | |
334 for data in instream: | |
335 hasher.update(data) | |
336 outfile.write(data) | |
337 | |
338 # Blecch: closing a file that somebody else opened is rude and | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
339 # wrong. But it's so darn convenient and practical! After all, |
15168 | 340 # outfile was opened just to copy and hash. |
341 outfile.close() | |
342 | |
343 return hasher.digest() | |
344 | |
345 def hashrepofile(repo, file): | |
346 return hashfile(repo.wjoin(file)) | |
347 | |
348 def hashfile(file): | |
349 if not os.path.exists(file): | |
350 return '' | |
351 hasher = util.sha1('') | |
352 fd = open(file, 'rb') | |
353 for data in blockstream(fd): | |
354 hasher.update(data) | |
355 fd.close() | |
356 return hasher.hexdigest() | |
357 | |
358 class limitreader(object): | |
359 def __init__(self, f, limit): | |
360 self.f = f | |
361 self.limit = limit | |
362 | |
363 def read(self, length): | |
364 if self.limit == 0: | |
365 return '' | |
366 length = length > self.limit and self.limit or length | |
367 self.limit -= length | |
368 return self.f.read(length) | |
369 | |
370 def close(self): | |
371 pass | |
372 | |
373 def blockstream(infile, blocksize=128 * 1024): | |
374 """Generator that yields blocks of data from infile and closes infile.""" | |
375 while True: | |
376 data = infile.read(blocksize) | |
377 if not data: | |
378 break | |
379 yield data | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
380 # same blecch as copyandhash() above |
15168 | 381 infile.close() |
382 | |
383 def readhash(filename): | |
384 rfile = open(filename, 'rb') | |
385 hash = rfile.read(40) | |
386 rfile.close() | |
387 if len(hash) < 40: | |
388 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') | |
389 % (filename, len(hash))) | |
390 return hash | |
391 | |
392 def writehash(hash, filename, executable): | |
393 util.makedirs(os.path.dirname(filename)) | |
15574
c9328c829cd9
largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents:
15572
diff
changeset
|
394 util.writefile(filename, hash + '\n') |
c9328c829cd9
largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents:
15572
diff
changeset
|
395 os.chmod(filename, getmode(executable)) |
15168 | 396 |
397 def getexecutable(filename): | |
398 mode = os.stat(filename).st_mode | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
399 return ((mode & stat.S_IXUSR) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
400 (mode & stat.S_IXGRP) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
401 (mode & stat.S_IXOTH)) |
15168 | 402 |
403 def getmode(executable): | |
404 if executable: | |
405 return 0755 | |
406 else: | |
407 return 0644 | |
408 | |
409 def urljoin(first, second, *arg): | |
410 def join(left, right): | |
411 if not left.endswith('/'): | |
412 left += '/' | |
413 if right.startswith('/'): | |
414 right = right[1:] | |
415 return left + right | |
416 | |
417 url = join(first, second) | |
418 for a in arg: | |
419 url = join(url, a) | |
420 return url | |
421 | |
422 def hexsha1(data): | |
423 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | |
424 object data""" | |
15347
799e56609ef6
largefiles: use util.sha1() instead of hashlib.sha1() everywhere
Thomas Arendsen Hein <thomas@intevation.de>
parents:
15333
diff
changeset
|
425 h = util.sha1() |
15168 | 426 for chunk in util.filechunkiter(data): |
427 h.update(chunk) | |
428 return h.hexdigest() | |
429 | |
430 def httpsendfile(ui, filename): | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15206
diff
changeset
|
431 return httpconnection.httpsendfile(ui, filename, 'rb') |
15168 | 432 |
433 def unixpath(path): | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15228
diff
changeset
|
434 '''Return a version of path normalized for use with the lfdirstate.''' |
15168 | 435 return os.path.normpath(path).replace(os.sep, '/') |
436 | |
437 def islfilesrepo(repo): | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
438 return ('largefiles' in repo.requirements and |
15319
9da7e96cd5c2
largefiles: remove redundant any_ function
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15317
diff
changeset
|
439 util.any(shortname + '/' in f[0] for f in repo.store.datafiles())) |
15168 | 440 |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
441 def mkstemp(repo, prefix): |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
442 '''Returns a file descriptor and a filename corresponding to a temporary |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
443 file in the repo's largefiles store.''' |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
444 path = repo.join(longname) |
15392
d7bfbc92a1c0
util: add a doctest for empty sha() calls
Matt Mackall <mpm@selenic.com>
parents:
15391
diff
changeset
|
445 util.makedirs(path) |
15391
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
446 return tempfile.mkstemp(prefix=prefix, dir=path) |
a5a6a9b7f3b9
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents:
15371
diff
changeset
|
447 |
15333
f37b71fec602
largefiles: py2.4 doesn't have BaseException
Matt Mackall <mpm@selenic.com>
parents:
15320
diff
changeset
|
448 class storeprotonotcapable(Exception): |
15168 | 449 def __init__(self, storetypes): |
450 self.storetypes = storetypes |