Mercurial > hg-stable
view mercurial/demandimport.py @ 24461:05ccfe6763f1
osutil: use getdirentriesattr on OS X if possible
This is a significant win for large repositories on OS X, especially with a
cold cache. Unfortunately we need to keep the lstat-based implementation around
for two reasons:
- Not all filesystems support this call.
- There's an edge case in which it's best to fall back to avoid a retry loop.
More about this in the comments.
The below tests are all performed on a Mac with an SSD running OS X 10.9, on a
repository with over 200k files. The results are best of 5 with simulated
best-effort conditions.
The gains with a hot cache are pretty impressive: 'hg status' goes from 5.18
seconds to 3.79 seconds.
However, a repository that large will probably already be using something like
hgwatchman [1], which helps much more (for this repo, 'hg status' with
hgwatchman is approximately 1 second). Where this really helps is when the
cache is cold [2]: hg status goes from 31.0 seconds to 9.66.
See http://lists.apple.com/archives/filesystem-dev/2014/Dec/msg00002.html for
some more discussion about this function.
This is based on a patch by Sean Farley <sean@farley.io>.
[1] https://bitbucket.org/facebook/hgwatchman
[2] There appears to be no easy way to clear the file cache (aka "vnodes") on
OS X short of rebooting. purge(8) purportedly does that but in my testing had
little effect. The workaround I came up with was to assume that vnode eviction
was LRU, make sure the kern.maxvnodes sysctl is smaller than the size of the
repository, then make sure we'd always miss the cache by running 'hg status' in
another clone of the repository before running it in the test repository.
author | Siddharth Agarwal <sid0@fb.com> |
---|---|
date | Wed, 25 Mar 2015 15:55:31 -0700 |
parents | 2205d00b6d2b |
children | 2e7804110b14 |
line wrap: on
line source
# demandimport.py - global demand-loading of modules for Mercurial # # Copyright 2006, 2007 Matt Mackall <mpm@selenic.com> # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. ''' demandimport - automatic demandloading of modules To enable this module, do: import demandimport; demandimport.enable() Imports of the following forms will be demand-loaded: import a, b.c import a.b as c from a import b,c # a will be loaded immediately These imports will not be delayed: from a import * b = __import__(a) ''' import __builtin__, os, sys _origimport = __import__ nothing = object() try: # Python 3 doesn't have relative imports nor level -1. level = -1 if sys.version_info[0] >= 3: level = 0 _origimport(__builtin__.__name__, {}, {}, None, level) except TypeError: # no level argument def _import(name, globals, locals, fromlist, level): "call _origimport with no level argument" return _origimport(name, globals, locals, fromlist) else: _import = _origimport def _hgextimport(importfunc, name, globals, *args): try: return importfunc(name, globals, *args) except ImportError: if not globals: raise # extensions are loaded with "hgext_" prefix hgextname = 'hgext_%s' % name nameroot = hgextname.split('.', 1)[0] contextroot = globals.get('__name__', '').split('.', 1)[0] if nameroot != contextroot: raise # retry to import with "hgext_" prefix return importfunc(hgextname, globals, *args) class _demandmod(object): """module demand-loader and proxy""" def __init__(self, name, globals, locals, level=level): if '.' in name: head, rest = name.split('.', 1) after = [rest] else: head = name after = [] object.__setattr__(self, "_data", (head, globals, locals, after, level)) object.__setattr__(self, "_module", None) def _extend(self, name): """add to the list of submodules to load""" self._data[3].append(name) def _load(self): if not self._module: head, globals, locals, after, level = self._data mod = _hgextimport(_import, head, globals, locals, None, level) # load submodules def subload(mod, p): h, t = p, None if '.' in p: h, t = p.split('.', 1) if getattr(mod, h, nothing) is nothing: setattr(mod, h, _demandmod(p, mod.__dict__, mod.__dict__)) elif t: subload(getattr(mod, h), t) for x in after: subload(mod, x) # are we in the locals dictionary still? if locals and locals.get(head) == self: locals[head] = mod object.__setattr__(self, "_module", mod) def __repr__(self): if self._module: return "<proxied module '%s'>" % self._data[0] return "<unloaded module '%s'>" % self._data[0] def __call__(self, *args, **kwargs): raise TypeError("%s object is not callable" % repr(self)) def __getattribute__(self, attr): if attr in ('_data', '_extend', '_load', '_module'): return object.__getattribute__(self, attr) self._load() return getattr(self._module, attr) def __setattr__(self, attr, val): self._load() setattr(self._module, attr, val) def _demandimport(name, globals=None, locals=None, fromlist=None, level=level): if not locals or name in ignore or fromlist == ('*',): # these cases we can't really delay return _hgextimport(_import, name, globals, locals, fromlist, level) elif not fromlist: # import a [as b] if '.' in name: # a.b base, rest = name.split('.', 1) # email.__init__ loading email.mime if globals and globals.get('__name__', None) == base: return _import(name, globals, locals, fromlist, level) # if a is already demand-loaded, add b to its submodule list if base in locals: if isinstance(locals[base], _demandmod): locals[base]._extend(rest) return locals[base] return _demandmod(name, globals, locals, level) else: if level != -1: # from . import b,c,d or from .a import b,c,d return _origimport(name, globals, locals, fromlist, level) # from a import b,c,d mod = _hgextimport(_origimport, name, globals, locals) # recurse down the module chain for comp in name.split('.')[1:]: if getattr(mod, comp, nothing) is nothing: setattr(mod, comp, _demandmod(comp, mod.__dict__, mod.__dict__)) mod = getattr(mod, comp) for x in fromlist: # set requested submodules for demand load if getattr(mod, x, nothing) is nothing: setattr(mod, x, _demandmod(x, mod.__dict__, locals)) return mod ignore = [ '_hashlib', '_xmlplus', 'fcntl', 'win32com.gen_py', '_winreg', # 2.7 mimetypes needs immediate ImportError 'pythoncom', # imported by tarfile, not available under Windows 'pwd', 'grp', # imported by profile, itself imported by hotshot.stats, # not available under Windows 'resource', # this trips up many extension authors 'gtk', # setuptools' pkg_resources.py expects "from __main__ import x" to # raise ImportError if x not defined '__main__', '_ssl', # conditional imports in the stdlib, issue1964 'rfc822', 'mimetools', # setuptools 8 expects this module to explode early when not on windows 'distutils.msvc9compiler' ] def isenabled(): return __builtin__.__import__ == _demandimport def enable(): "enable global demand-loading of modules" if os.environ.get('HGDEMANDIMPORT') != 'disable': __builtin__.__import__ = _demandimport def disable(): "disable global demand-loading of modules" __builtin__.__import__ = _origimport