Mercurial > hg
view mercurial/narrowspec.py @ 39270:37e56607cbb9
lfs: add a progress bar when searching for blobs to upload
The search itself can take an extreme amount of time if there are a lot of
revisions involved. I've got a local repo that took 6 minutes to push 1850
commits, and 60% of that time was spent here (there are ~70K files):
\ 58.1% wrapper.py: extractpointers line 297: pointers = extractpointers(...
| 57.7% wrapper.py: pointersfromctx line 352: for p in pointersfromctx(ct...
| 57.4% wrapper.py: pointerfromctx line 397: p = pointerfromctx(ctx, f, ...
\ 38.7% context.py: __contains__ line 368: if f not in ctx:
| 38.7% util.py: __get__ line 82: return key in self._manifest
| 38.7% context.py: _manifest line 1416: result = self.func(obj)
| 38.7% manifest.py: read line 472: return self._manifestctx.re...
\ 25.6% revlog.py: revision line 1562: text = rl.revision(self._node)
\ 12.8% revlog.py: _chunks line 2217: bins = self._chunks(chain, ...
| 12.0% revlog.py: decompressline 2112: ladd(decomp(buffer(data, ch...
\ 7.8% revlog.py: checkhash line 2232: self.checkhash(text, node, ...
| 7.8% revlog.py: hash line 2315: if node != self.hash(text, ...
| 7.8% revlog.py: hash line 2242: return hash(text, p1, p2)
\ 12.0% manifest.py: __init__ line 1565: self._data = manifestdict(t...
\ 16.8% context.py: filenode line 378: if not _islfs(fctx.filelog(...
| 15.7% util.py: __get__ line 706: return self._filelog
| 14.8% context.py: _filelog line 1416: result = self.func(obj)
| 14.8% localrepo.py: file line 629: return self._repo.file(self...
| 14.8% filelog.py: __init__ line 1134: return filelog.filelog(self...
| 14.5% revlog.py: __init__ line 24: censorable=True)
author | Matt Harbison <matt_harbison@yahoo.com> |
---|---|
date | Fri, 24 Aug 2018 17:45:46 -0400 |
parents | 576eef1ab43d |
children | 2675d561f5cb |
line wrap: on
line source
# narrowspec.py - methods for working with a narrow view of a repository # # Copyright 2017 Google, Inc. # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. from __future__ import absolute_import import errno from .i18n import _ from . import ( error, match as matchmod, repository, sparse, util, ) FILENAME = 'narrowspec' def parseserverpatterns(text): """Parses the narrowspec format that's returned by the server.""" includepats = set() excludepats = set() # We get one entry per line, in the format "<key> <value>". # It's OK for value to contain other spaces. for kp in (l.split(' ', 1) for l in text.splitlines()): if len(kp) != 2: raise error.Abort(_('Invalid narrowspec pattern line: "%s"') % kp) key = kp[0] pat = kp[1] if key == 'include': includepats.add(pat) elif key == 'exclude': excludepats.add(pat) else: raise error.Abort(_('Invalid key "%s" in server response') % key) return includepats, excludepats def normalizesplitpattern(kind, pat): """Returns the normalized version of a pattern and kind. Returns a tuple with the normalized kind and normalized pattern. """ pat = pat.rstrip('/') _validatepattern(pat) return kind, pat def _numlines(s): """Returns the number of lines in s, including ending empty lines.""" # We use splitlines because it is Unicode-friendly and thus Python 3 # compatible. However, it does not count empty lines at the end, so trick # it by adding a character at the end. return len((s + 'x').splitlines()) def _validatepattern(pat): """Validates the pattern and aborts if it is invalid. Patterns are stored in the narrowspec as newline-separated POSIX-style bytestring paths. There's no escaping. """ # We use newlines as separators in the narrowspec file, so don't allow them # in patterns. if _numlines(pat) > 1: raise error.Abort(_('newlines are not allowed in narrowspec paths')) components = pat.split('/') if '.' in components or '..' in components: raise error.Abort(_('"." and ".." are not allowed in narrowspec paths')) def normalizepattern(pattern, defaultkind='path'): """Returns the normalized version of a text-format pattern. If the pattern has no kind, the default will be added. """ kind, pat = matchmod._patsplit(pattern, defaultkind) return '%s:%s' % normalizesplitpattern(kind, pat) def parsepatterns(pats): """Parses a list of patterns into a typed pattern set.""" return set(normalizepattern(p) for p in pats) def format(includes, excludes): output = '[include]\n' for i in sorted(includes - excludes): output += i + '\n' output += '[exclude]\n' for e in sorted(excludes): output += e + '\n' return output def match(root, include=None, exclude=None): if not include: # Passing empty include and empty exclude to matchmod.match() # gives a matcher that matches everything, so explicitly use # the nevermatcher. return matchmod.never(root, '') return matchmod.match(root, '', [], include=include or [], exclude=exclude or []) def needsexpansion(includes): return [i for i in includes if i.startswith('include:')] def load(repo): try: spec = repo.svfs.read(FILENAME) except IOError as e: # Treat "narrowspec does not exist" the same as "narrowspec file exists # and is empty". if e.errno == errno.ENOENT: return set(), set() raise # maybe we should care about the profiles returned too includepats, excludepats, profiles = sparse.parseconfig(repo.ui, spec, 'narrow') if profiles: raise error.Abort(_("including other spec files using '%include' is not" " suported in narrowspec")) return includepats, excludepats def save(repo, includepats, excludepats): spec = format(includepats, excludepats) repo.svfs.write(FILENAME, spec) def savebackup(repo, backupname): if repository.NARROW_REQUIREMENT not in repo.requirements: return vfs = repo.vfs vfs.tryunlink(backupname) util.copyfile(repo.svfs.join(FILENAME), vfs.join(backupname), hardlink=True) def restorebackup(repo, backupname): if repository.NARROW_REQUIREMENT not in repo.requirements: return util.rename(repo.vfs.join(backupname), repo.svfs.join(FILENAME)) def clearbackup(repo, backupname): if repository.NARROW_REQUIREMENT not in repo.requirements: return repo.vfs.unlink(backupname) def restrictpatterns(req_includes, req_excludes, repo_includes, repo_excludes): r""" Restricts the patterns according to repo settings, results in a logical AND operation :param req_includes: requested includes :param req_excludes: requested excludes :param repo_includes: repo includes :param repo_excludes: repo excludes :return: include patterns, exclude patterns, and invalid include patterns. >>> restrictpatterns({'f1','f2'}, {}, ['f1'], []) (set(['f1']), {}, []) >>> restrictpatterns({'f1'}, {}, ['f1','f2'], []) (set(['f1']), {}, []) >>> restrictpatterns({'f1/fc1', 'f3/fc3'}, {}, ['f1','f2'], []) (set(['f1/fc1']), {}, []) >>> restrictpatterns({'f1_fc1'}, {}, ['f1','f2'], []) ([], set(['path:.']), []) >>> restrictpatterns({'f1/../f2/fc2'}, {}, ['f1','f2'], []) (set(['f2/fc2']), {}, []) >>> restrictpatterns({'f1/../f3/fc3'}, {}, ['f1','f2'], []) ([], set(['path:.']), []) >>> restrictpatterns({'f1/$non_exitent_var'}, {}, ['f1','f2'], []) (set(['f1/$non_exitent_var']), {}, []) """ res_excludes = set(req_excludes) res_excludes.update(repo_excludes) invalid_includes = [] if not req_includes: res_includes = set(repo_includes) elif 'path:.' not in repo_includes: res_includes = [] for req_include in req_includes: req_include = util.expandpath(util.normpath(req_include)) if req_include in repo_includes: res_includes.append(req_include) continue valid = False for repo_include in repo_includes: if req_include.startswith(repo_include + '/'): valid = True res_includes.append(req_include) break if not valid: invalid_includes.append(req_include) if len(res_includes) == 0: res_excludes = {'path:.'} else: res_includes = set(res_includes) else: res_includes = set(req_includes) return res_includes, res_excludes, invalid_includes