Mercurial > hg-stable
comparison hgext/sparse.py @ 33289:abd7dedbaa36
sparse: vendor Facebook-developed extension
Facebook has developed an extension to enable "sparse" checkouts -
a working directory with a subset of files. This feature is a critical
component in enabling repositories to scale to infinite number of
files while retaining reasonable performance. It's worth noting
that sparse checkout is only one possible solution to this problem:
another is virtual filesystems that realize files on first access.
But given that virtual filesystems may not be accessible to all
users, sparse checkout is necessary as a fallback.
Per mailing list discussion at
https://www.mercurial-scm.org/pipermail/mercurial-devel/2017-March/095868.html
we want to add sparse checkout to the Mercurial distribution via
roughly the following mechanism:
1. Vendor extension as-is with minimal modifications (this patch)
2. Refactor extension so it is more clearly experimental and inline
with Mercurial practices
3. Move code from extension into core where possible
4. Drop experimental labeling and/or move feature into core
after sign-off from narrow clone feature owners
This commit essentially copies the sparse extension and tests
from revision 71e0a2aeca92a4078fe1b8c76e32c88ff1929737 of the
https://bitbucket.org/facebook/hg-experimental repository.
A list of modifications made as part of vendoring is as follows:
* "EXPERIMENTAL" added to module docstring
* Imports were changed to match Mercurial style conventions
* "testedwith" value was updated to core Mercurial special value and
comment boilerplate was inserted
* A "clone_sparse" function was renamed to "clonesparse" to appease
the style checker
* Paths to the sparse extension in tests reflect built-in location
* test-sparse-extensions.t was renamed to test-sparse-fsmonitor.t
and references to "simplecache" were removed. The test always skips
because it isn't trivial to run it given the way we currently run
fsmonitor tests
* A double empty line was removed from test-sparse-profiles.t
There are aspects of the added code that are obviously not ideal.
The goal is to make a minimal number of modifications as part of
the vendoring to make it easier to track changes from the original
implementation. Refactoring will occur in subsequent patches.
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Sat, 01 Jul 2017 10:43:29 -0700 |
parents | |
children | cd1c275c9482 |
comparison
equal
deleted
inserted
replaced
33288:f08a178adadf | 33289:abd7dedbaa36 |
---|---|
1 # sparse.py - allow sparse checkouts of the working directory | |
2 # | |
3 # Copyright 2014 Facebook, Inc. | |
4 # | |
5 # This software may be used and distributed according to the terms of the | |
6 # GNU General Public License version 2 or any later version. | |
7 | |
8 """allow sparse checkouts of the working directory (EXPERIMENTAL) | |
9 """ | |
10 | |
11 from __future__ import absolute_import | |
12 | |
13 import collections | |
14 import hashlib | |
15 import os | |
16 | |
17 from mercurial.i18n import _ | |
18 from mercurial.node import nullid | |
19 from mercurial import ( | |
20 cmdutil, | |
21 commands, | |
22 context, | |
23 dirstate, | |
24 error, | |
25 extensions, | |
26 hg, | |
27 localrepo, | |
28 match as matchmod, | |
29 merge as mergemod, | |
30 registrar, | |
31 util, | |
32 ) | |
33 | |
34 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | |
35 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | |
36 # be specifying the version(s) of Mercurial they are tested with, or | |
37 # leave the attribute unspecified. | |
38 testedwith = 'ships-with-hg-core' | |
39 | |
40 cmdtable = {} | |
41 command = registrar.command(cmdtable) | |
42 | |
43 def uisetup(ui): | |
44 _setupupdates(ui) | |
45 _setupcommit(ui) | |
46 | |
47 def extsetup(ui): | |
48 _setupclone(ui) | |
49 _setuplog(ui) | |
50 _setupadd(ui) | |
51 _setupdirstate(ui) | |
52 # if fsmonitor is enabled, tell it to use our hash function | |
53 try: | |
54 fsmonitor = extensions.find('fsmonitor') | |
55 def _hashignore(orig, ignore): | |
56 return _hashmatcher(ignore) | |
57 extensions.wrapfunction(fsmonitor, '_hashignore', _hashignore) | |
58 except KeyError: | |
59 pass | |
60 # do the same for hgwatchman, old name | |
61 try: | |
62 hgwatchman = extensions.find('hgwatchman') | |
63 def _hashignore(orig, ignore): | |
64 return _hashmatcher(ignore) | |
65 extensions.wrapfunction(hgwatchman, '_hashignore', _hashignore) | |
66 except KeyError: | |
67 pass | |
68 | |
69 def reposetup(ui, repo): | |
70 if not util.safehasattr(repo, 'dirstate'): | |
71 return | |
72 | |
73 _wraprepo(ui, repo) | |
74 | |
75 def replacefilecache(cls, propname, replacement): | |
76 """Replace a filecache property with a new class. This allows changing the | |
77 cache invalidation condition.""" | |
78 origcls = cls | |
79 assert callable(replacement) | |
80 while cls is not object: | |
81 if propname in cls.__dict__: | |
82 orig = cls.__dict__[propname] | |
83 setattr(cls, propname, replacement(orig)) | |
84 break | |
85 cls = cls.__bases__[0] | |
86 | |
87 if cls is object: | |
88 raise AttributeError(_("type '%s' has no property '%s'") % (origcls, | |
89 propname)) | |
90 | |
91 def _setupupdates(ui): | |
92 def _calculateupdates(orig, repo, wctx, mctx, ancestors, branchmerge, *arg, | |
93 **kwargs): | |
94 """Filter updates to only lay out files that match the sparse rules. | |
95 """ | |
96 actions, diverge, renamedelete = orig(repo, wctx, mctx, ancestors, | |
97 branchmerge, *arg, **kwargs) | |
98 | |
99 if not util.safehasattr(repo, 'sparsematch'): | |
100 return actions, diverge, renamedelete | |
101 | |
102 files = set() | |
103 prunedactions = {} | |
104 oldrevs = [pctx.rev() for pctx in wctx.parents()] | |
105 oldsparsematch = repo.sparsematch(*oldrevs) | |
106 | |
107 if branchmerge: | |
108 # If we're merging, use the wctx filter, since we're merging into | |
109 # the wctx. | |
110 sparsematch = repo.sparsematch(wctx.parents()[0].rev()) | |
111 else: | |
112 # If we're updating, use the target context's filter, since we're | |
113 # moving to the target context. | |
114 sparsematch = repo.sparsematch(mctx.rev()) | |
115 | |
116 temporaryfiles = [] | |
117 for file, action in actions.iteritems(): | |
118 type, args, msg = action | |
119 files.add(file) | |
120 if sparsematch(file): | |
121 prunedactions[file] = action | |
122 elif type == 'm': | |
123 temporaryfiles.append(file) | |
124 prunedactions[file] = action | |
125 elif branchmerge: | |
126 if type != 'k': | |
127 temporaryfiles.append(file) | |
128 prunedactions[file] = action | |
129 elif type == 'f': | |
130 prunedactions[file] = action | |
131 elif file in wctx: | |
132 prunedactions[file] = ('r', args, msg) | |
133 | |
134 if len(temporaryfiles) > 0: | |
135 ui.status(_("temporarily included %d file(s) in the sparse checkout" | |
136 " for merging\n") % len(temporaryfiles)) | |
137 repo.addtemporaryincludes(temporaryfiles) | |
138 | |
139 # Add the new files to the working copy so they can be merged, etc | |
140 actions = [] | |
141 message = 'temporarily adding to sparse checkout' | |
142 wctxmanifest = repo[None].manifest() | |
143 for file in temporaryfiles: | |
144 if file in wctxmanifest: | |
145 fctx = repo[None][file] | |
146 actions.append((file, (fctx.flags(), False), message)) | |
147 | |
148 typeactions = collections.defaultdict(list) | |
149 typeactions['g'] = actions | |
150 mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], | |
151 False) | |
152 | |
153 dirstate = repo.dirstate | |
154 for file, flags, msg in actions: | |
155 dirstate.normal(file) | |
156 | |
157 profiles = repo.getactiveprofiles() | |
158 changedprofiles = profiles & files | |
159 # If an active profile changed during the update, refresh the checkout. | |
160 # Don't do this during a branch merge, since all incoming changes should | |
161 # have been handled by the temporary includes above. | |
162 if changedprofiles and not branchmerge: | |
163 mf = mctx.manifest() | |
164 for file in mf: | |
165 old = oldsparsematch(file) | |
166 new = sparsematch(file) | |
167 if not old and new: | |
168 flags = mf.flags(file) | |
169 prunedactions[file] = ('g', (flags, False), '') | |
170 elif old and not new: | |
171 prunedactions[file] = ('r', [], '') | |
172 | |
173 return prunedactions, diverge, renamedelete | |
174 | |
175 extensions.wrapfunction(mergemod, 'calculateupdates', _calculateupdates) | |
176 | |
177 def _update(orig, repo, node, branchmerge, *args, **kwargs): | |
178 results = orig(repo, node, branchmerge, *args, **kwargs) | |
179 | |
180 # If we're updating to a location, clean up any stale temporary includes | |
181 # (ex: this happens during hg rebase --abort). | |
182 if not branchmerge and util.safehasattr(repo, 'sparsematch'): | |
183 repo.prunetemporaryincludes() | |
184 return results | |
185 | |
186 extensions.wrapfunction(mergemod, 'update', _update) | |
187 | |
188 def _setupcommit(ui): | |
189 def _refreshoncommit(orig, self, node): | |
190 """Refresh the checkout when commits touch .hgsparse | |
191 """ | |
192 orig(self, node) | |
193 repo = self._repo | |
194 if util.safehasattr(repo, 'getsparsepatterns'): | |
195 ctx = repo[node] | |
196 _, _, profiles = repo.getsparsepatterns(ctx.rev()) | |
197 if set(profiles) & set(ctx.files()): | |
198 origstatus = repo.status() | |
199 origsparsematch = repo.sparsematch() | |
200 _refresh(repo.ui, repo, origstatus, origsparsematch, True) | |
201 | |
202 repo.prunetemporaryincludes() | |
203 | |
204 extensions.wrapfunction(context.committablectx, 'markcommitted', | |
205 _refreshoncommit) | |
206 | |
207 def _setuplog(ui): | |
208 entry = commands.table['^log|history'] | |
209 entry[1].append(('', 'sparse', None, | |
210 "limit to changesets affecting the sparse checkout")) | |
211 | |
212 def _logrevs(orig, repo, opts): | |
213 revs = orig(repo, opts) | |
214 if opts.get('sparse'): | |
215 sparsematch = repo.sparsematch() | |
216 def ctxmatch(rev): | |
217 ctx = repo[rev] | |
218 return any(f for f in ctx.files() if sparsematch(f)) | |
219 revs = revs.filter(ctxmatch) | |
220 return revs | |
221 extensions.wrapfunction(cmdutil, '_logrevs', _logrevs) | |
222 | |
223 def _clonesparsecmd(orig, ui, repo, *args, **opts): | |
224 include_pat = opts.get('include') | |
225 exclude_pat = opts.get('exclude') | |
226 enableprofile_pat = opts.get('enable_profile') | |
227 include = exclude = enableprofile = False | |
228 if include_pat: | |
229 pat = include_pat | |
230 include = True | |
231 if exclude_pat: | |
232 pat = exclude_pat | |
233 exclude = True | |
234 if enableprofile_pat: | |
235 pat = enableprofile_pat | |
236 enableprofile = True | |
237 if sum([include, exclude, enableprofile]) > 1: | |
238 raise error.Abort(_("too many flags specified.")) | |
239 if include or exclude or enableprofile: | |
240 def clonesparse(orig, self, node, overwrite, *args, **kwargs): | |
241 _config(self.ui, self.unfiltered(), pat, {}, include=include, | |
242 exclude=exclude, enableprofile=enableprofile) | |
243 return orig(self, node, overwrite, *args, **kwargs) | |
244 extensions.wrapfunction(hg, 'updaterepo', clonesparse) | |
245 return orig(ui, repo, *args, **opts) | |
246 | |
247 def _setupclone(ui): | |
248 entry = commands.table['^clone'] | |
249 entry[1].append(('', 'enable-profile', [], | |
250 'enable a sparse profile')) | |
251 entry[1].append(('', 'include', [], | |
252 'include sparse pattern')) | |
253 entry[1].append(('', 'exclude', [], | |
254 'exclude sparse pattern')) | |
255 extensions.wrapcommand(commands.table, 'clone', _clonesparsecmd) | |
256 | |
257 def _setupadd(ui): | |
258 entry = commands.table['^add'] | |
259 entry[1].append(('s', 'sparse', None, | |
260 'also include directories of added files in sparse config')) | |
261 | |
262 def _add(orig, ui, repo, *pats, **opts): | |
263 if opts.get('sparse'): | |
264 dirs = set() | |
265 for pat in pats: | |
266 dirname, basename = util.split(pat) | |
267 dirs.add(dirname) | |
268 _config(ui, repo, list(dirs), opts, include=True) | |
269 return orig(ui, repo, *pats, **opts) | |
270 | |
271 extensions.wrapcommand(commands.table, 'add', _add) | |
272 | |
273 def _setupdirstate(ui): | |
274 """Modify the dirstate to prevent stat'ing excluded files, | |
275 and to prevent modifications to files outside the checkout. | |
276 """ | |
277 | |
278 def _dirstate(orig, repo): | |
279 dirstate = orig(repo) | |
280 dirstate.repo = repo | |
281 return dirstate | |
282 extensions.wrapfunction( | |
283 localrepo.localrepository.dirstate, 'func', _dirstate) | |
284 | |
285 # The atrocity below is needed to wrap dirstate._ignore. It is a cached | |
286 # property, which means normal function wrapping doesn't work. | |
287 class ignorewrapper(object): | |
288 def __init__(self, orig): | |
289 self.orig = orig | |
290 self.origignore = None | |
291 self.func = None | |
292 self.sparsematch = None | |
293 | |
294 def __get__(self, obj, type=None): | |
295 repo = obj.repo | |
296 origignore = self.orig.__get__(obj) | |
297 if not util.safehasattr(repo, 'sparsematch'): | |
298 return origignore | |
299 | |
300 sparsematch = repo.sparsematch() | |
301 if self.sparsematch != sparsematch or self.origignore != origignore: | |
302 self.func = unionmatcher([origignore, | |
303 negatematcher(sparsematch)]) | |
304 self.sparsematch = sparsematch | |
305 self.origignore = origignore | |
306 return self.func | |
307 | |
308 def __set__(self, obj, value): | |
309 return self.orig.__set__(obj, value) | |
310 | |
311 def __delete__(self, obj): | |
312 return self.orig.__delete__(obj) | |
313 | |
314 replacefilecache(dirstate.dirstate, '_ignore', ignorewrapper) | |
315 | |
316 # dirstate.rebuild should not add non-matching files | |
317 def _rebuild(orig, self, parent, allfiles, changedfiles=None): | |
318 if util.safehasattr(self.repo, 'sparsematch'): | |
319 matcher = self.repo.sparsematch() | |
320 allfiles = allfiles.matches(matcher) | |
321 if changedfiles: | |
322 changedfiles = [f for f in changedfiles if matcher(f)] | |
323 | |
324 if changedfiles is not None: | |
325 # In _rebuild, these files will be deleted from the dirstate | |
326 # when they are not found to be in allfiles | |
327 dirstatefilestoremove = set(f for f in self if not matcher(f)) | |
328 changedfiles = dirstatefilestoremove.union(changedfiles) | |
329 | |
330 return orig(self, parent, allfiles, changedfiles) | |
331 extensions.wrapfunction(dirstate.dirstate, 'rebuild', _rebuild) | |
332 | |
333 # Prevent adding files that are outside the sparse checkout | |
334 editfuncs = ['normal', 'add', 'normallookup', 'copy', 'remove', 'merge'] | |
335 hint = _('include file with `hg sparse --include <pattern>` or use ' + | |
336 '`hg add -s <file>` to include file directory while adding') | |
337 for func in editfuncs: | |
338 def _wrapper(orig, self, *args): | |
339 repo = self.repo | |
340 if util.safehasattr(repo, 'sparsematch'): | |
341 dirstate = repo.dirstate | |
342 sparsematch = repo.sparsematch() | |
343 for f in args: | |
344 if (f is not None and not sparsematch(f) and | |
345 f not in dirstate): | |
346 raise error.Abort(_("cannot add '%s' - it is outside " | |
347 "the sparse checkout") % f, | |
348 hint=hint) | |
349 return orig(self, *args) | |
350 extensions.wrapfunction(dirstate.dirstate, func, _wrapper) | |
351 | |
352 def _wraprepo(ui, repo): | |
353 class SparseRepo(repo.__class__): | |
354 def readsparseconfig(self, raw): | |
355 """Takes a string sparse config and returns the includes, | |
356 excludes, and profiles it specified. | |
357 """ | |
358 includes = set() | |
359 excludes = set() | |
360 current = includes | |
361 profiles = [] | |
362 for line in raw.split('\n'): | |
363 line = line.strip() | |
364 if not line or line.startswith('#'): | |
365 # empty or comment line, skip | |
366 continue | |
367 elif line.startswith('%include '): | |
368 line = line[9:].strip() | |
369 if line: | |
370 profiles.append(line) | |
371 elif line == '[include]': | |
372 if current != includes: | |
373 raise error.Abort(_('.hg/sparse cannot have includes ' + | |
374 'after excludes')) | |
375 continue | |
376 elif line == '[exclude]': | |
377 current = excludes | |
378 elif line: | |
379 if line.strip().startswith('/'): | |
380 self.ui.warn(_('warning: sparse profile cannot use' + | |
381 ' paths starting with /, ignoring %s\n') | |
382 % line) | |
383 continue | |
384 current.add(line) | |
385 | |
386 return includes, excludes, profiles | |
387 | |
388 def getsparsepatterns(self, rev): | |
389 """Returns the include/exclude patterns specified by the | |
390 given rev. | |
391 """ | |
392 if not self.vfs.exists('sparse'): | |
393 return set(), set(), [] | |
394 if rev is None: | |
395 raise error.Abort(_("cannot parse sparse patterns from " + | |
396 "working copy")) | |
397 | |
398 raw = self.vfs.read('sparse') | |
399 includes, excludes, profiles = self.readsparseconfig(raw) | |
400 | |
401 ctx = self[rev] | |
402 if profiles: | |
403 visited = set() | |
404 while profiles: | |
405 profile = profiles.pop() | |
406 if profile in visited: | |
407 continue | |
408 visited.add(profile) | |
409 | |
410 try: | |
411 raw = self.getrawprofile(profile, rev) | |
412 except error.ManifestLookupError: | |
413 msg = ( | |
414 "warning: sparse profile '%s' not found " | |
415 "in rev %s - ignoring it\n" % (profile, ctx)) | |
416 if self.ui.configbool( | |
417 'sparse', 'missingwarning', True): | |
418 self.ui.warn(msg) | |
419 else: | |
420 self.ui.debug(msg) | |
421 continue | |
422 pincludes, pexcludes, subprofs = \ | |
423 self.readsparseconfig(raw) | |
424 includes.update(pincludes) | |
425 excludes.update(pexcludes) | |
426 for subprofile in subprofs: | |
427 profiles.append(subprofile) | |
428 | |
429 profiles = visited | |
430 | |
431 if includes: | |
432 includes.add('.hg*') | |
433 return includes, excludes, profiles | |
434 | |
435 def getrawprofile(self, profile, changeid): | |
436 try: | |
437 simplecache = extensions.find('simplecache') | |
438 node = self[changeid].hex() | |
439 def func(): | |
440 return self.filectx(profile, changeid=changeid).data() | |
441 key = 'sparseprofile:%s:%s' % (profile.replace('/', '__'), node) | |
442 return simplecache.memoize(func, key, | |
443 simplecache.stringserializer, self.ui) | |
444 except KeyError: | |
445 return self.filectx(profile, changeid=changeid).data() | |
446 | |
447 def sparsechecksum(self, filepath): | |
448 fh = open(filepath) | |
449 return hashlib.sha1(fh.read()).hexdigest() | |
450 | |
451 def _sparsesignature(self, includetemp=True): | |
452 """Returns the signature string representing the contents of the | |
453 current project sparse configuration. This can be used to cache the | |
454 sparse matcher for a given set of revs.""" | |
455 signaturecache = self.signaturecache | |
456 signature = signaturecache.get('signature') | |
457 if includetemp: | |
458 tempsignature = signaturecache.get('tempsignature') | |
459 else: | |
460 tempsignature = 0 | |
461 | |
462 if signature is None or (includetemp and tempsignature is None): | |
463 signature = 0 | |
464 try: | |
465 sparsepath = self.vfs.join('sparse') | |
466 signature = self.sparsechecksum(sparsepath) | |
467 except (OSError, IOError): | |
468 pass | |
469 signaturecache['signature'] = signature | |
470 | |
471 tempsignature = 0 | |
472 if includetemp: | |
473 try: | |
474 tempsparsepath = self.vfs.join('tempsparse') | |
475 tempsignature = self.sparsechecksum(tempsparsepath) | |
476 except (OSError, IOError): | |
477 pass | |
478 signaturecache['tempsignature'] = tempsignature | |
479 return '%s %s' % (str(signature), str(tempsignature)) | |
480 | |
481 def invalidatecaches(self): | |
482 self.invalidatesignaturecache() | |
483 return super(SparseRepo, self).invalidatecaches() | |
484 | |
485 def invalidatesignaturecache(self): | |
486 self.signaturecache.clear() | |
487 | |
488 def sparsematch(self, *revs, **kwargs): | |
489 """Returns the sparse match function for the given revs. | |
490 | |
491 If multiple revs are specified, the match function is the union | |
492 of all the revs. | |
493 | |
494 `includetemp` is used to indicate if the temporarily included file | |
495 should be part of the matcher. | |
496 """ | |
497 if not revs or revs == (None,): | |
498 revs = [self.changelog.rev(node) for node in | |
499 self.dirstate.parents() if node != nullid] | |
500 | |
501 includetemp = kwargs.get('includetemp', True) | |
502 signature = self._sparsesignature(includetemp=includetemp) | |
503 | |
504 key = '%s %s' % (str(signature), ' '.join([str(r) for r in revs])) | |
505 | |
506 result = self.sparsecache.get(key, None) | |
507 if result: | |
508 return result | |
509 | |
510 matchers = [] | |
511 for rev in revs: | |
512 try: | |
513 includes, excludes, profiles = self.getsparsepatterns(rev) | |
514 | |
515 if includes or excludes: | |
516 # Explicitly include subdirectories of includes so | |
517 # status will walk them down to the actual include. | |
518 subdirs = set() | |
519 for include in includes: | |
520 dirname = os.path.dirname(include) | |
521 # basename is used to avoid issues with absolute | |
522 # paths (which on Windows can include the drive). | |
523 while os.path.basename(dirname): | |
524 subdirs.add(dirname) | |
525 dirname = os.path.dirname(dirname) | |
526 | |
527 matcher = matchmod.match(self.root, '', [], | |
528 include=includes, exclude=excludes, | |
529 default='relpath') | |
530 if subdirs: | |
531 matcher = forceincludematcher(matcher, subdirs) | |
532 matchers.append(matcher) | |
533 except IOError: | |
534 pass | |
535 | |
536 result = None | |
537 if not matchers: | |
538 result = matchmod.always(self.root, '') | |
539 elif len(matchers) == 1: | |
540 result = matchers[0] | |
541 else: | |
542 result = unionmatcher(matchers) | |
543 | |
544 if kwargs.get('includetemp', True): | |
545 tempincludes = self.gettemporaryincludes() | |
546 result = forceincludematcher(result, tempincludes) | |
547 | |
548 self.sparsecache[key] = result | |
549 | |
550 return result | |
551 | |
552 def getactiveprofiles(self): | |
553 revs = [self.changelog.rev(node) for node in | |
554 self.dirstate.parents() if node != nullid] | |
555 | |
556 activeprofiles = set() | |
557 for rev in revs: | |
558 _, _, profiles = self.getsparsepatterns(rev) | |
559 activeprofiles.update(profiles) | |
560 | |
561 return activeprofiles | |
562 | |
563 def writesparseconfig(self, include, exclude, profiles): | |
564 raw = '%s[include]\n%s\n[exclude]\n%s\n' % ( | |
565 ''.join(['%%include %s\n' % p for p in sorted(profiles)]), | |
566 '\n'.join(sorted(include)), | |
567 '\n'.join(sorted(exclude))) | |
568 self.vfs.write("sparse", raw) | |
569 self.invalidatesignaturecache() | |
570 | |
571 def addtemporaryincludes(self, files): | |
572 includes = self.gettemporaryincludes() | |
573 for file in files: | |
574 includes.add(file) | |
575 self._writetemporaryincludes(includes) | |
576 | |
577 def gettemporaryincludes(self): | |
578 existingtemp = set() | |
579 if self.vfs.exists('tempsparse'): | |
580 raw = self.vfs.read('tempsparse') | |
581 existingtemp.update(raw.split('\n')) | |
582 return existingtemp | |
583 | |
584 def _writetemporaryincludes(self, includes): | |
585 raw = '\n'.join(sorted(includes)) | |
586 self.vfs.write('tempsparse', raw) | |
587 self.invalidatesignaturecache() | |
588 | |
589 def prunetemporaryincludes(self): | |
590 if repo.vfs.exists('tempsparse'): | |
591 origstatus = self.status() | |
592 modified, added, removed, deleted, a, b, c = origstatus | |
593 if modified or added or removed or deleted: | |
594 # Still have pending changes. Don't bother trying to prune. | |
595 return | |
596 | |
597 sparsematch = self.sparsematch(includetemp=False) | |
598 dirstate = self.dirstate | |
599 actions = [] | |
600 dropped = [] | |
601 tempincludes = self.gettemporaryincludes() | |
602 for file in tempincludes: | |
603 if file in dirstate and not sparsematch(file): | |
604 message = 'dropping temporarily included sparse files' | |
605 actions.append((file, None, message)) | |
606 dropped.append(file) | |
607 | |
608 typeactions = collections.defaultdict(list) | |
609 typeactions['r'] = actions | |
610 mergemod.applyupdates(self, typeactions, self[None], self['.'], | |
611 False) | |
612 | |
613 # Fix dirstate | |
614 for file in dropped: | |
615 dirstate.drop(file) | |
616 | |
617 self.vfs.unlink('tempsparse') | |
618 self.invalidatesignaturecache() | |
619 msg = _("cleaned up %d temporarily added file(s) from the " | |
620 "sparse checkout\n") | |
621 ui.status(msg % len(tempincludes)) | |
622 | |
623 if 'dirstate' in repo._filecache: | |
624 repo.dirstate.repo = repo | |
625 repo.sparsecache = {} | |
626 repo.signaturecache = {} | |
627 repo.__class__ = SparseRepo | |
628 | |
629 @command('^sparse', [ | |
630 ('I', 'include', False, _('include files in the sparse checkout')), | |
631 ('X', 'exclude', False, _('exclude files in the sparse checkout')), | |
632 ('d', 'delete', False, _('delete an include/exclude rule')), | |
633 ('f', 'force', False, _('allow changing rules even with pending changes')), | |
634 ('', 'enable-profile', False, _('enables the specified profile')), | |
635 ('', 'disable-profile', False, _('disables the specified profile')), | |
636 ('', 'import-rules', False, _('imports rules from a file')), | |
637 ('', 'clear-rules', False, _('clears local include/exclude rules')), | |
638 ('', 'refresh', False, _('updates the working after sparseness changes')), | |
639 ('', 'reset', False, _('makes the repo full again')), | |
640 ] + commands.templateopts, | |
641 _('[--OPTION] PATTERN...')) | |
642 def sparse(ui, repo, *pats, **opts): | |
643 """make the current checkout sparse, or edit the existing checkout | |
644 | |
645 The sparse command is used to make the current checkout sparse. | |
646 This means files that don't meet the sparse condition will not be | |
647 written to disk, or show up in any working copy operations. It does | |
648 not affect files in history in any way. | |
649 | |
650 Passing no arguments prints the currently applied sparse rules. | |
651 | |
652 --include and --exclude are used to add and remove files from the sparse | |
653 checkout. The effects of adding an include or exclude rule are applied | |
654 immediately. If applying the new rule would cause a file with pending | |
655 changes to be added or removed, the command will fail. Pass --force to | |
656 force a rule change even with pending changes (the changes on disk will | |
657 be preserved). | |
658 | |
659 --delete removes an existing include/exclude rule. The effects are | |
660 immediate. | |
661 | |
662 --refresh refreshes the files on disk based on the sparse rules. This is | |
663 only necessary if .hg/sparse was changed by hand. | |
664 | |
665 --enable-profile and --disable-profile accept a path to a .hgsparse file. | |
666 This allows defining sparse checkouts and tracking them inside the | |
667 repository. This is useful for defining commonly used sparse checkouts for | |
668 many people to use. As the profile definition changes over time, the sparse | |
669 checkout will automatically be updated appropriately, depending on which | |
670 changeset is checked out. Changes to .hgsparse are not applied until they | |
671 have been committed. | |
672 | |
673 --import-rules accepts a path to a file containing rules in the .hgsparse | |
674 format, allowing you to add --include, --exclude and --enable-profile rules | |
675 in bulk. Like the --include, --exclude and --enable-profile switches, the | |
676 changes are applied immediately. | |
677 | |
678 --clear-rules removes all local include and exclude rules, while leaving | |
679 any enabled profiles in place. | |
680 | |
681 Returns 0 if editing the sparse checkout succeeds. | |
682 """ | |
683 include = opts.get('include') | |
684 exclude = opts.get('exclude') | |
685 force = opts.get('force') | |
686 enableprofile = opts.get('enable_profile') | |
687 disableprofile = opts.get('disable_profile') | |
688 importrules = opts.get('import_rules') | |
689 clearrules = opts.get('clear_rules') | |
690 delete = opts.get('delete') | |
691 refresh = opts.get('refresh') | |
692 reset = opts.get('reset') | |
693 count = sum([include, exclude, enableprofile, disableprofile, delete, | |
694 importrules, refresh, clearrules, reset]) | |
695 if count > 1: | |
696 raise error.Abort(_("too many flags specified")) | |
697 | |
698 if count == 0: | |
699 if repo.vfs.exists('sparse'): | |
700 ui.status(repo.vfs.read("sparse") + "\n") | |
701 temporaryincludes = repo.gettemporaryincludes() | |
702 if temporaryincludes: | |
703 ui.status(_("Temporarily Included Files (for merge/rebase):\n")) | |
704 ui.status(("\n".join(temporaryincludes) + "\n")) | |
705 else: | |
706 ui.status(_('repo is not sparse\n')) | |
707 return | |
708 | |
709 if include or exclude or delete or reset or enableprofile or disableprofile: | |
710 _config(ui, repo, pats, opts, include=include, exclude=exclude, | |
711 reset=reset, delete=delete, enableprofile=enableprofile, | |
712 disableprofile=disableprofile, force=force) | |
713 | |
714 if importrules: | |
715 _import(ui, repo, pats, opts, force=force) | |
716 | |
717 if clearrules: | |
718 _clear(ui, repo, pats, force=force) | |
719 | |
720 if refresh: | |
721 try: | |
722 wlock = repo.wlock() | |
723 fcounts = map( | |
724 len, | |
725 _refresh(ui, repo, repo.status(), repo.sparsematch(), force)) | |
726 _verbose_output(ui, opts, 0, 0, 0, *fcounts) | |
727 finally: | |
728 wlock.release() | |
729 | |
730 def _config(ui, repo, pats, opts, include=False, exclude=False, reset=False, | |
731 delete=False, enableprofile=False, disableprofile=False, | |
732 force=False): | |
733 """ | |
734 Perform a sparse config update. Only one of the kwargs may be specified. | |
735 """ | |
736 wlock = repo.wlock() | |
737 try: | |
738 oldsparsematch = repo.sparsematch() | |
739 | |
740 if repo.vfs.exists('sparse'): | |
741 raw = repo.vfs.read('sparse') | |
742 oldinclude, oldexclude, oldprofiles = map( | |
743 set, repo.readsparseconfig(raw)) | |
744 else: | |
745 oldinclude = set() | |
746 oldexclude = set() | |
747 oldprofiles = set() | |
748 | |
749 try: | |
750 if reset: | |
751 newinclude = set() | |
752 newexclude = set() | |
753 newprofiles = set() | |
754 else: | |
755 newinclude = set(oldinclude) | |
756 newexclude = set(oldexclude) | |
757 newprofiles = set(oldprofiles) | |
758 | |
759 oldstatus = repo.status() | |
760 | |
761 if any(pat.startswith('/') for pat in pats): | |
762 ui.warn(_('warning: paths cannot start with /, ignoring: %s\n') | |
763 % ([pat for pat in pats if pat.startswith('/')])) | |
764 elif include: | |
765 newinclude.update(pats) | |
766 elif exclude: | |
767 newexclude.update(pats) | |
768 elif enableprofile: | |
769 newprofiles.update(pats) | |
770 elif disableprofile: | |
771 newprofiles.difference_update(pats) | |
772 elif delete: | |
773 newinclude.difference_update(pats) | |
774 newexclude.difference_update(pats) | |
775 | |
776 repo.writesparseconfig(newinclude, newexclude, newprofiles) | |
777 fcounts = map( | |
778 len, _refresh(ui, repo, oldstatus, oldsparsematch, force)) | |
779 | |
780 profilecount = (len(newprofiles - oldprofiles) - | |
781 len(oldprofiles - newprofiles)) | |
782 includecount = (len(newinclude - oldinclude) - | |
783 len(oldinclude - newinclude)) | |
784 excludecount = (len(newexclude - oldexclude) - | |
785 len(oldexclude - newexclude)) | |
786 _verbose_output( | |
787 ui, opts, profilecount, includecount, excludecount, *fcounts) | |
788 except Exception: | |
789 repo.writesparseconfig(oldinclude, oldexclude, oldprofiles) | |
790 raise | |
791 finally: | |
792 wlock.release() | |
793 | |
794 def _import(ui, repo, files, opts, force=False): | |
795 with repo.wlock(): | |
796 # load union of current active profile | |
797 revs = [repo.changelog.rev(node) for node in | |
798 repo.dirstate.parents() if node != nullid] | |
799 | |
800 # read current configuration | |
801 raw = '' | |
802 if repo.vfs.exists('sparse'): | |
803 raw = repo.vfs.read('sparse') | |
804 oincludes, oexcludes, oprofiles = repo.readsparseconfig(raw) | |
805 includes, excludes, profiles = map( | |
806 set, (oincludes, oexcludes, oprofiles)) | |
807 | |
808 # all active rules | |
809 aincludes, aexcludes, aprofiles = set(), set(), set() | |
810 for rev in revs: | |
811 rincludes, rexcludes, rprofiles = repo.getsparsepatterns(rev) | |
812 aincludes.update(rincludes) | |
813 aexcludes.update(rexcludes) | |
814 aprofiles.update(rprofiles) | |
815 | |
816 # import rules on top; only take in rules that are not yet | |
817 # part of the active rules. | |
818 changed = False | |
819 for file in files: | |
820 with util.posixfile(util.expandpath(file)) as importfile: | |
821 iincludes, iexcludes, iprofiles = repo.readsparseconfig( | |
822 importfile.read()) | |
823 oldsize = len(includes) + len(excludes) + len(profiles) | |
824 includes.update(iincludes - aincludes) | |
825 excludes.update(iexcludes - aexcludes) | |
826 profiles.update(set(iprofiles) - aprofiles) | |
827 if len(includes) + len(excludes) + len(profiles) > oldsize: | |
828 changed = True | |
829 | |
830 profilecount = includecount = excludecount = 0 | |
831 fcounts = (0, 0, 0) | |
832 | |
833 if changed: | |
834 profilecount = len(profiles - aprofiles) | |
835 includecount = len(includes - aincludes) | |
836 excludecount = len(excludes - aexcludes) | |
837 | |
838 oldstatus = repo.status() | |
839 oldsparsematch = repo.sparsematch() | |
840 repo.writesparseconfig(includes, excludes, profiles) | |
841 | |
842 try: | |
843 fcounts = map( | |
844 len, _refresh(ui, repo, oldstatus, oldsparsematch, force)) | |
845 except Exception: | |
846 repo.writesparseconfig(oincludes, oexcludes, oprofiles) | |
847 raise | |
848 | |
849 _verbose_output(ui, opts, profilecount, includecount, excludecount, | |
850 *fcounts) | |
851 | |
852 def _clear(ui, repo, files, force=False): | |
853 with repo.wlock(): | |
854 raw = '' | |
855 if repo.vfs.exists('sparse'): | |
856 raw = repo.vfs.read('sparse') | |
857 includes, excludes, profiles = repo.readsparseconfig(raw) | |
858 | |
859 if includes or excludes: | |
860 oldstatus = repo.status() | |
861 oldsparsematch = repo.sparsematch() | |
862 repo.writesparseconfig(set(), set(), profiles) | |
863 _refresh(ui, repo, oldstatus, oldsparsematch, force) | |
864 | |
865 def _refresh(ui, repo, origstatus, origsparsematch, force): | |
866 """Refreshes which files are on disk by comparing the old status and | |
867 sparsematch with the new sparsematch. | |
868 | |
869 Will raise an exception if a file with pending changes is being excluded | |
870 or included (unless force=True). | |
871 """ | |
872 modified, added, removed, deleted, unknown, ignored, clean = origstatus | |
873 | |
874 # Verify there are no pending changes | |
875 pending = set() | |
876 pending.update(modified) | |
877 pending.update(added) | |
878 pending.update(removed) | |
879 sparsematch = repo.sparsematch() | |
880 abort = False | |
881 for file in pending: | |
882 if not sparsematch(file): | |
883 ui.warn(_("pending changes to '%s'\n") % file) | |
884 abort = not force | |
885 if abort: | |
886 raise error.Abort(_("could not update sparseness due to " + | |
887 "pending changes")) | |
888 | |
889 # Calculate actions | |
890 dirstate = repo.dirstate | |
891 ctx = repo['.'] | |
892 added = [] | |
893 lookup = [] | |
894 dropped = [] | |
895 mf = ctx.manifest() | |
896 files = set(mf) | |
897 | |
898 actions = {} | |
899 | |
900 for file in files: | |
901 old = origsparsematch(file) | |
902 new = sparsematch(file) | |
903 # Add files that are newly included, or that don't exist in | |
904 # the dirstate yet. | |
905 if (new and not old) or (old and new and not file in dirstate): | |
906 fl = mf.flags(file) | |
907 if repo.wvfs.exists(file): | |
908 actions[file] = ('e', (fl,), '') | |
909 lookup.append(file) | |
910 else: | |
911 actions[file] = ('g', (fl, False), '') | |
912 added.append(file) | |
913 # Drop files that are newly excluded, or that still exist in | |
914 # the dirstate. | |
915 elif (old and not new) or (not old and not new and file in dirstate): | |
916 dropped.append(file) | |
917 if file not in pending: | |
918 actions[file] = ('r', [], '') | |
919 | |
920 # Verify there are no pending changes in newly included files | |
921 abort = False | |
922 for file in lookup: | |
923 ui.warn(_("pending changes to '%s'\n") % file) | |
924 abort = not force | |
925 if abort: | |
926 raise error.Abort(_("cannot change sparseness due to " + | |
927 "pending changes (delete the files or use --force " + | |
928 "to bring them back dirty)")) | |
929 | |
930 # Check for files that were only in the dirstate. | |
931 for file, state in dirstate.iteritems(): | |
932 if not file in files: | |
933 old = origsparsematch(file) | |
934 new = sparsematch(file) | |
935 if old and not new: | |
936 dropped.append(file) | |
937 | |
938 # Apply changes to disk | |
939 typeactions = dict((m, []) for m in 'a f g am cd dc r dm dg m e k'.split()) | |
940 for f, (m, args, msg) in actions.iteritems(): | |
941 if m not in typeactions: | |
942 typeactions[m] = [] | |
943 typeactions[m].append((f, args, msg)) | |
944 mergemod.applyupdates(repo, typeactions, repo[None], repo['.'], False) | |
945 | |
946 # Fix dirstate | |
947 for file in added: | |
948 dirstate.normal(file) | |
949 | |
950 for file in dropped: | |
951 dirstate.drop(file) | |
952 | |
953 for file in lookup: | |
954 # File exists on disk, and we're bringing it back in an unknown state. | |
955 dirstate.normallookup(file) | |
956 | |
957 return added, dropped, lookup | |
958 | |
959 def _verbose_output(ui, opts, profilecount, includecount, excludecount, added, | |
960 dropped, lookup): | |
961 """Produce --verbose and templatable output | |
962 | |
963 This specifically enables -Tjson, providing machine-readable stats on how | |
964 the sparse profile changed. | |
965 | |
966 """ | |
967 with ui.formatter('sparse', opts) as fm: | |
968 fm.startitem() | |
969 fm.condwrite(ui.verbose, 'profiles_added', 'Profile # change: %d\n', | |
970 profilecount) | |
971 fm.condwrite(ui.verbose, 'include_rules_added', | |
972 'Include rule # change: %d\n', includecount) | |
973 fm.condwrite(ui.verbose, 'exclude_rules_added', | |
974 'Exclude rule # change: %d\n', excludecount) | |
975 # In 'plain' verbose mode, mergemod.applyupdates already outputs what | |
976 # files are added or removed outside of the templating formatter | |
977 # framework. No point in repeating ourselves in that case. | |
978 if not fm.isplain(): | |
979 fm.condwrite(ui.verbose, 'files_added', 'Files added: %d\n', | |
980 added) | |
981 fm.condwrite(ui.verbose, 'files_dropped', 'Files dropped: %d\n', | |
982 dropped) | |
983 fm.condwrite(ui.verbose, 'files_conflicting', | |
984 'Files conflicting: %d\n', lookup) | |
985 | |
986 class forceincludematcher(object): | |
987 """A matcher that returns true for any of the forced includes before testing | |
988 against the actual matcher.""" | |
989 def __init__(self, matcher, includes): | |
990 self._matcher = matcher | |
991 self._includes = includes | |
992 | |
993 def __call__(self, value): | |
994 return value in self._includes or self._matcher(value) | |
995 | |
996 def always(self): | |
997 return False | |
998 | |
999 def files(self): | |
1000 return [] | |
1001 | |
1002 def isexact(self): | |
1003 return False | |
1004 | |
1005 def anypats(self): | |
1006 return True | |
1007 | |
1008 def prefix(self): | |
1009 return False | |
1010 | |
1011 def hash(self): | |
1012 sha1 = hashlib.sha1() | |
1013 sha1.update(_hashmatcher(self._matcher)) | |
1014 for include in sorted(self._includes): | |
1015 sha1.update(include + '\0') | |
1016 return sha1.hexdigest() | |
1017 | |
1018 class unionmatcher(object): | |
1019 """A matcher that is the union of several matchers.""" | |
1020 def __init__(self, matchers): | |
1021 self._matchers = matchers | |
1022 | |
1023 def __call__(self, value): | |
1024 for match in self._matchers: | |
1025 if match(value): | |
1026 return True | |
1027 return False | |
1028 | |
1029 def always(self): | |
1030 return False | |
1031 | |
1032 def files(self): | |
1033 return [] | |
1034 | |
1035 def isexact(self): | |
1036 return False | |
1037 | |
1038 def anypats(self): | |
1039 return True | |
1040 | |
1041 def prefix(self): | |
1042 return False | |
1043 | |
1044 def hash(self): | |
1045 sha1 = hashlib.sha1() | |
1046 for m in self._matchers: | |
1047 sha1.update(_hashmatcher(m)) | |
1048 return sha1.hexdigest() | |
1049 | |
1050 class negatematcher(object): | |
1051 def __init__(self, matcher): | |
1052 self._matcher = matcher | |
1053 | |
1054 def __call__(self, value): | |
1055 return not self._matcher(value) | |
1056 | |
1057 def always(self): | |
1058 return False | |
1059 | |
1060 def files(self): | |
1061 return [] | |
1062 | |
1063 def isexact(self): | |
1064 return False | |
1065 | |
1066 def anypats(self): | |
1067 return True | |
1068 | |
1069 def hash(self): | |
1070 sha1 = hashlib.sha1() | |
1071 sha1.update('negate') | |
1072 sha1.update(_hashmatcher(self._matcher)) | |
1073 return sha1.hexdigest() | |
1074 | |
1075 def _hashmatcher(matcher): | |
1076 if util.safehasattr(matcher, 'hash'): | |
1077 return matcher.hash() | |
1078 | |
1079 sha1 = hashlib.sha1() | |
1080 sha1.update(repr(matcher)) | |
1081 return sha1.hexdigest() |