mercurial/demandimport.py

branches: avoid unnecessary changectx.branch() calls

This requires reading from the changelog, which can be costly over NFS.

Note that this does not totally remove reading from the changelog; we
still do that when calling changectx.closesbranch(). That call will be
removed in a later patch.

Running hg branches on the PyPy repo (with 996) over a busy NFS server,
before this change:

$ time hg --profile branches > /dev/null
   CallCount    Recursive     Total(s)    Inline(s) module:lineno(function)
        2042            0      2.2827      2.2827   <open>
        2036            0      0.9840      0.9840   <method 'close' of 'file' objects>
        2036            0      0.0464      0.0464   <method 'read' of 'file' objects>
        5233            0      0.1985      0.0453   mercurial.repoview:161(changelog)
       10462            0      0.0791      0.0314       mercurial.changelog:133(tip)
        5233            0      0.0388      0.0176       mercurial.localrepo:26(__get__)
       10462            0      0.0250      0.0126       <len>
        5233            0      0.0059      0.0039       mercurial.repoview:112(filterrevs)
       10462            0      0.0029      0.0029       <hash>
        2034            0      0.0444      0.0444   <method 'seek' of 'file' objects>
        5340            0      0.0390      0.0390   mercurial.revlog:296(rev)
        2582            0      0.0371      0.0371   <zlib.decompress>
        3155            0      0.1963      0.0366   mercurial.context:202(__init__)
        3155            0      0.1238      0.0306       mercurial.repoview:161(changelog)
        3155            0      0.0261      0.0080       mercurial.changelog:183(rev)
        9465            0      0.0061      0.0061       <isinstance>
        1096            0      0.0023      0.0023       <binascii.unhexlify>
        4251            0      0.0014      0.0014       <len>
        2059            0      3.7341      0.0332   mercurial.changelog:270(read)
        2059            0      3.6304      0.0307       mercurial.revlog:907(revision)
        2057            0      0.0262      0.0137       mercurial.changelog:28(decodeextra)
        4118            0      0.0094      0.0094       <method 'split' of 'str' objects>
        4118            0      0.0270      0.0048       mercurial.encoding:61(tolocal)
        2059            0      0.0040      0.0040       <method 'index' of 'str' objects>
       10462            0      0.0791      0.0314   mercurial.changelog:133(tip)
       10462            0      0.0289      0.0207       mercurial.changelog:190(node)
       10462            0      0.0188      0.0091       <len>
       52433        20932      0.0478      0.0310   <len>
       20932            0      0.0221      0.0168       mercurial.revlog:262(__len__)
        2059            0      3.6304      0.0307   mercurial.revlog:907(revision)

real    0m4.361s
user    0m0.986s
sys     0m0.237s

After this change:

$ time hg --profile branches > /dev/null
   CallCount    Recursive     Total(s)    Inline(s) module:lineno(function)
        1069            0      1.1098      1.1098   <open>
        1063            0      0.4865      0.4865   <method 'close' of 'file' objects>
        4122            0      0.1811      0.0404   mercurial.repoview:161(changelog)
        8240            0      0.0712      0.0272       mercurial.changelog:133(tip)
        4122            0      0.0378      0.0177       mercurial.localrepo:26(__get__)
        8240            0      0.0221      0.0115       <len>
        4122            0      0.0057      0.0033       mercurial.repoview:112(filterrevs)
        8240            0      0.0025      0.0025       <hash>
        3029            0      0.1979      0.0371   mercurial.context:202(__init__)
        3029            0      0.1278      0.0310       mercurial.repoview:161(changelog)
        3029            0      0.0230      0.0081       mercurial.changelog:183(rev)
        9087            0      0.0061      0.0061       <isinstance>
        1096            0      0.0026      0.0026       <binascii.unhexlify>
        4125            0      0.0014      0.0014       <len>
        4229            0      0.0337      0.0337   mercurial.revlog:296(rev)
        1061            0      0.0296      0.0296   <method 'seek' of 'file' objects>
        1063            0      0.0292      0.0292   <method 'read' of 'file' objects>
        8240            0      0.0712      0.0272   mercurial.changelog:133(tip)
        8240            0      0.0271      0.0196       mercurial.changelog:190(node)
        8240            0      0.0169      0.0083       <len>
       40476        16488      0.0422      0.0271   <len>
       16488            0      0.0193      0.0152       mercurial.revlog:262(__len__)
        1342            0      0.0241      0.0241   <zlib.decompress>
        9445            0      0.0336      0.0224   mercurial.changelog:190(node)
        9445            0      0.0112      0.0112       mercurial.revlog:317(node)
        1074            0      1.9102      0.0224   mercurial.changelog:270(read)
        1074            0      1.8397      0.0202       mercurial.revlog:907(revision)
        1073            0      0.0187      0.0099       mercurial.changelog:28(decodeextra)
        2148            0      0.0061      0.0061       <method 'split' of 'str' objects>
        2148            0      0.0184      0.0034       mercurial.encoding:61(tolocal)

real    0m2.402s
user    0m0.735s
sys     0m0.177s

# demandimport.py - global demand-loading of modules for Mercurial
#
# Copyright 2006, 2007 Matt Mackall <[e-mail]>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

'''
demandimport - automatic demandloading of modules

To enable this module, do:

  import demandimport; demandimport.enable()

Imports of the following forms will be demand-loaded:

  import a, b.c
  import a.b as c
  from a import b,c # a will be loaded immediately

These imports will not be delayed:

  from a import *
  b = __import__(a)
'''

import __builtin__
_origimport = __import__

nothing = object()

try:
    _origimport(__builtin__.__name__, {}, {}, None, -1)
except TypeError: # no level argument
    def _import(name, globals, locals, fromlist, level):
        "call _origimport with no level argument"
        return _origimport(name, globals, locals, fromlist)
else:
    _import = _origimport

def _hgextimport(importfunc, name, globals, *args):
    try:
        return importfunc(name, globals, *args)
    except ImportError:
        if not globals:
            raise
        # extensions are loaded with "hgext_" prefix
        hgextname = 'hgext_%s' % name
        nameroot = hgextname.split('.', 1)[0]
        contextroot = globals.get('__name__', '').split('.', 1)[0]
        if nameroot != contextroot:
            raise
        # retry to import with "hgext_" prefix
        return importfunc(hgextname, globals, *args)

class _demandmod(object):
    """module demand-loader and proxy"""
    def __init__(self, name, globals, locals, level=-1):
        if '.' in name:
            head, rest = name.split('.', 1)
            after = [rest]
        else:
            head = name
            after = []
        object.__setattr__(self, "_data",
                           (head, globals, locals, after, level))
        object.__setattr__(self, "_module", None)
    def _extend(self, name):
        """add to the list of submodules to load"""
        self._data[3].append(name)
    def _load(self):
        if not self._module:
            head, globals, locals, after, level = self._data
            mod = _hgextimport(_import, head, globals, locals, None, level)
            # load submodules
            def subload(mod, p):
                h, t = p, None
                if '.' in p:
                    h, t = p.split('.', 1)
                if getattr(mod, h, nothing) is nothing:
                    setattr(mod, h, _demandmod(p, mod.__dict__, mod.__dict__))
                elif t:
                    subload(getattr(mod, h), t)

            for x in after:
                subload(mod, x)

            # are we in the locals dictionary still?
            if locals and locals.get(head) == self:
                locals[head] = mod
            object.__setattr__(self, "_module", mod)

    def __repr__(self):
        if self._module:
            return "<proxied module '%s'>" % self._data[0]
        return "<unloaded module '%s'>" % self._data[0]
    def __call__(self, *args, **kwargs):
        raise TypeError("%s object is not callable" % repr(self))
    def __getattribute__(self, attr):
        if attr in ('_data', '_extend', '_load', '_module'):
            return object.__getattribute__(self, attr)
        self._load()
        return getattr(self._module, attr)
    def __setattr__(self, attr, val):
        self._load()
        setattr(self._module, attr, val)

def _demandimport(name, globals=None, locals=None, fromlist=None, level=-1):
    if not locals or name in ignore or fromlist == ('*',):
        # these cases we can't really delay
        return _hgextimport(_import, name, globals, locals, fromlist, level)
    elif not fromlist:
        # import a [as b]
        if '.' in name: # a.b
            base, rest = name.split('.', 1)
            # email.__init__ loading email.mime
            if globals and globals.get('__name__', None) == base:
                return _import(name, globals, locals, fromlist, level)
            # if a is already demand-loaded, add b to its submodule list
            if base in locals:
                if isinstance(locals[base], _demandmod):
                    locals[base]._extend(rest)
                return locals[base]
        return _demandmod(name, globals, locals, level)
    else:
        if level != -1:
            # from . import b,c,d or from .a import b,c,d
            return _origimport(name, globals, locals, fromlist, level)
        # from a import b,c,d
        mod = _hgextimport(_origimport, name, globals, locals)
        # recurse down the module chain
        for comp in name.split('.')[1:]:
            if getattr(mod, comp, nothing) is nothing:
                setattr(mod, comp, _demandmod(comp, mod.__dict__, mod.__dict__))
            mod = getattr(mod, comp)
        for x in fromlist:
            # set requested submodules for demand load
            if getattr(mod, x, nothing) is nothing:
                setattr(mod, x, _demandmod(x, mod.__dict__, locals))
        return mod

ignore = [
    '_hashlib',
    '_xmlplus',
    'fcntl',
    'win32com.gen_py',
    '_winreg', # 2.7 mimetypes needs immediate ImportError
    'pythoncom',
    # imported by tarfile, not available under Windows
    'pwd',
    'grp',
    # imported by profile, itself imported by hotshot.stats,
    # not available under Windows
    'resource',
    # this trips up many extension authors
    'gtk',
    # setuptools' pkg_resources.py expects "from __main__ import x" to
    # raise ImportError if x not defined
    '__main__',
    '_ssl', # conditional imports in the stdlib, issue1964
    'rfc822',
    'mimetools',
    ]

def enable():
    "enable global demand-loading of modules"
    __builtin__.__import__ = _demandimport

def disable():
    "disable global demand-loading of modules"
    __builtin__.__import__ = _origimport