annotate mercurial/profiling.py @ 33499:0407a51b9d8c

codemod: register core configitems using a script

This is done by a script [2] using RedBaron [1], a tool designed for doing
code refactoring. All "default" values are decided by the script and are
strongly consistent with the existing code.

There are 2 changes done manually to fix tests:

  [warn] mercurial/exchange.py: experimental.bundle2-output-capture: default needs manual removal
  [warn] mercurial/localrepo.py: experimental.hook-track-tags: default needs manual removal

Since RedBaron is not confident about how to indent things [2].

[1]: https://github.com/PyCQA/redbaron
[2]: https://github.com/PyCQA/redbaron/issues/100
[3]:

#!/usr/bin/env python
# codemod_configitems.py - codemod tool to fill configitems
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import, print_function

import os
import sys

import redbaron

def readpath(path):
    with open(path) as f:
        return f.read()

def writepath(path, content):
    with open(path, 'w') as f:
        f.write(content)

_configmethods = {'config', 'configbool', 'configint', 'configbytes',
                  'configlist', 'configdate'}

def extractstring(rnode):
    """get the string from a RedBaron string or call_argument node"""
    while rnode.type != 'string':
        rnode = rnode.value
    return rnode.value[1:-1]  # unquote, "'str'" -> "str"

def uiconfigitems(red):
    """match *.ui.config* pattern, yield (node, method, args, section, name)"""
    for node in red.find_all('atomtrailers'):
        entry = None
        try:
            obj = node[-3].value
            method = node[-2].value
            args = node[-1]
            section = args[0].value
            name = args[1].value
            if (obj in ('ui', 'self') and method in _configmethods
                and section.type == 'string' and name.type == 'string'):
                entry = (node, method, args, extractstring(section),
                         extractstring(name))
        except Exception:
            pass
        else:
            if entry:
                yield entry

def coreconfigitems(red):
    """match coreconfigitem(...) pattern, yield (node, args, section, name)"""
    for node in red.find_all('atomtrailers'):
        entry = None
        try:
            args = node[1]
            section = args[0].value
            name = args[1].value
            if (node[0].value == 'coreconfigitem' and section.type == 'string'
                and name.type == 'string'):
                entry = (node, args, extractstring(section),
                         extractstring(name))
        except Exception:
            pass
        else:
            if entry:
                yield entry

def registercoreconfig(cfgred, section, name, defaultrepr):
    """insert coreconfigitem to cfgred AST

    section and name are plain string, defaultrepr is a string
    """
    # find a place to insert the "coreconfigitem" item
    entries = list(coreconfigitems(cfgred))
    for node, args, nodesection, nodename in reversed(entries):
        if (nodesection, nodename) < (section, name):
            # insert after this entry
            node.insert_after(
                'coreconfigitem(%r, %r,\n'
                '    default=%s,\n'
                ')' % (section, name, defaultrepr))
            return

def main(argv):
    if not argv:
        print('Usage: codemod_configitems.py FILES\n'
              'For example, FILES could be "{hgext,mercurial}/*/**.py"')
    dirname = os.path.dirname
    reporoot = dirname(dirname(dirname(os.path.abspath(__file__))))

    # register configitems to this destination
    cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py')
    cfgred = redbaron.RedBaron(readpath(cfgpath))

    # state about what to do
    registered = set((s, n) for n, a, s, n in coreconfigitems(cfgred))
    toregister = {} # {(section, name): defaultrepr}
    coreconfigs = set() # {(section, name)}, whether it's used in core

    # first loop: scan all files before taking any action
    for i, path in enumerate(argv):
        print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
        iscore = ('mercurial' in path) and ('hgext' not in path)
        red = redbaron.RedBaron(readpath(path))
        # find all repo.ui.config* and ui.config* calls, and collect their
        # section, name and default value information.
        for node, method, args, section, name in uiconfigitems(red):
            if section == 'web':
                # [web] section has some weirdness, ignore them for now
                continue
            defaultrepr = None
            key = (section, name)
            if len(args) == 2:
                if key in registered:
                    continue
                if method == 'configlist':
                    defaultrepr = 'list'
                elif method == 'configbool':
                    defaultrepr = 'False'
                else:
                    defaultrepr = 'None'
            elif len(args) >= 3 and (args[2].target is None or
                                     args[2].target.value == 'default'):
                # try to understand the "default" value
                dnode = args[2].value
                if dnode.type == 'name':
                    if dnode.value in {'None', 'True', 'False'}:
                        defaultrepr = dnode.value
                elif dnode.type == 'string':
                    defaultrepr = repr(dnode.value[1:-1])
                elif dnode.type in ('int', 'float'):
                    defaultrepr = dnode.value
            # inconsistent default
            if key in toregister and toregister[key] != defaultrepr:
                defaultrepr = None
            # interesting to rewrite
            if key not in registered:
                if defaultrepr is None:
                    print('[note] %s: %s.%s: unsupported default'
                          % (path, section, name))
                    registered.add(key) # skip checking it again
                else:
                    toregister[key] = defaultrepr
                    if iscore:
                        coreconfigs.add(key)

    # second loop: rewrite files given "toregister" result
    for path in argv:
        # reconstruct redbaron - trade CPU for memory
        red = redbaron.RedBaron(readpath(path))
        changed = False
        for node, method, args, section, name in uiconfigitems(red):
            key = (section, name)
            defaultrepr = toregister.get(key)
            if defaultrepr is None or key not in coreconfigs:
                continue
            if len(args) >= 3 and (args[2].target is None or
                                   args[2].target.value == 'default'):
                try:
                    del args[2]
                    changed = True
                except Exception:
                    # redbaron fails to do the rewrite due to indentation
                    # see https://github.com/PyCQA/redbaron/issues/100
                    print('[warn] %s: %s.%s: default needs manual removal'
                          % (path, section, name))
            if key not in registered:
                print('registering %s.%s' % (section, name))
                registercoreconfig(cfgred, section, name, defaultrepr)
                registered.add(key)
        if changed:
            print('updating %s' % path)
            writepath(path, red.dumps())

    if toregister:
        print('updating configitems.py')
        writepath(cfgpath, cfgred.dumps())

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))

# profiling.py - profiling functions
#
# Copyright 2016 Gregory Szorc <[e-mail]>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import, print_function

import contextlib

from .i18n import _
from . import (
    encoding,
    error,
    extensions,
    util,
)

def _loadprofiler(ui, profiler):
    """load profiler extension. return profile method, or None on failure"""
    extname = profiler
    extensions.loadall(ui, whitelist=[extname])
    try:
        mod = extensions.find(extname)
    except KeyError:
        return None
    else:
        return getattr(mod, 'profile', None)

@contextlib.contextmanager
def lsprofile(ui, fp):
    format = ui.config('profiling', 'format')
    field = ui.config('profiling', 'sort')
    limit = ui.configint('profiling', 'limit')
    climit = ui.configint('profiling', 'nested')

    if format not in ['text', 'kcachegrind']:
        ui.warn(_("unrecognized profiling format '%s'"
                    " - Ignored\n") % format)
        format = 'text'

    try:
        from . import lsprof
    except ImportError:
        raise error.Abort(_(
            'lsprof not available - install from '
            'http://codespeak.net/svn/user/arigo/hack/misc/lsprof/'))
    p = lsprof.Profiler()
    p.enable(subcalls=True)
    try:
        yield
    finally:
        p.disable()

        if format == 'kcachegrind':
            from . import lsprofcalltree
            calltree = lsprofcalltree.KCacheGrind(p)
            calltree.output(fp)
        else:
            # format == 'text'
            stats = lsprof.Stats(p.getstats())
            stats.sort(field)
            stats.pprint(limit=limit, file=fp, climit=climit)

@contextlib.contextmanager
def flameprofile(ui, fp):
    try:
        from flamegraph import flamegraph
    except ImportError:
        raise error.Abort(_(
            'flamegraph not available - install from '
            'https://github.com/evanhempel/python-flamegraph'))
    # developer config: profiling.freq
    freq = ui.configint('profiling', 'freq')
    filter_ = None
    collapse_recursion = True
    thread = flamegraph.ProfileThread(fp, 1.0 / freq,
                                      filter_, collapse_recursion)
    start_time = util.timer()
    try:
        thread.start()
        yield
    finally:
        thread.stop()
        thread.join()
        print('Collected %d stack frames (%d unique) in %2.2f seconds.' % (
            util.timer() - start_time, thread.num_frames(),
            thread.num_frames(unique=True)))

@contextlib.contextmanager
def statprofile(ui, fp):
    from . import statprof

    freq = ui.configint('profiling', 'freq')
    if freq > 0:
        # Cannot reset when profiler is already active. So silently no-op.
        if statprof.state.profile_level == 0:
            statprof.reset(freq)
    else:
        ui.warn(_("invalid sampling frequency '%s' - ignoring\n") % freq)

    statprof.start(mechanism='thread')

    try:
        yield
    finally:
        data = statprof.stop()

        profformat = ui.config('profiling', 'statformat')

        formats = {
            'byline': statprof.DisplayFormats.ByLine,
            'bymethod': statprof.DisplayFormats.ByMethod,
            'hotpath': statprof.DisplayFormats.Hotpath,
            'json': statprof.DisplayFormats.Json,
            'chrome': statprof.DisplayFormats.Chrome,
        }

        if profformat in formats:
            displayformat = formats[profformat]
        else:
            ui.warn(_('unknown profiler output format: %s\n') % profformat)
            displayformat = statprof.DisplayFormats.Hotpath

        kwargs = {}

        def fraction(s):
            if isinstance(s, (float, int)):
                return float(s)
            if s.endswith('%'):
                v = float(s[:-1]) / 100
            else:
                v = float(s)
            if 0 <= v <= 1:
                return v
            raise ValueError(s)

        if profformat == 'chrome':
            showmin = ui.configwith(fraction, 'profiling', 'showmin', 0.005)
            showmax = ui.configwith(fraction, 'profiling', 'showmax', 0.999)
            kwargs.update(minthreshold=showmin, maxthreshold=showmax)
        elif profformat == 'hotpath':
            # inconsistent config: profiling.showmin
            limit = ui.configwith(fraction, 'profiling', 'showmin', 0.05)
            kwargs['limit'] = limit

        statprof.display(fp, data=data, format=displayformat, **kwargs)

class profile(object):
    """Start profiling.

    Profiling is active when the context manager is active. When the context
    manager exits, profiling results will be written to the configured output.
    """
    def __init__(self, ui, enabled=True):
        self._ui = ui
        self._output = None
        self._fp = None
        self._fpdoclose = True
        self._profiler = None
        self._enabled = enabled
        self._entered = False
        self._started = False

    def __enter__(self):
        self._entered = True
        if self._enabled:
            self.start()
        return self

    def start(self):
        """Start profiling.

        The profiling will stop at the context exit.

        If the profiler was already started, this has no effect."""
        if not self._entered:
            raise error.ProgrammingError()
        if self._started:
            return
        self._started = True
        profiler = encoding.environ.get('HGPROF')
        proffn = None
        if profiler is None:
            profiler = self._ui.config('profiling', 'type', default='stat')
        if profiler not in ('ls', 'stat', 'flame'):
            # try load profiler from extension with the same name
            proffn = _loadprofiler(self._ui, profiler)
            if proffn is None:
                self._ui.warn(_("unrecognized profiler '%s' - ignored\n")
                              % profiler)
                profiler = 'stat'

        self._output = self._ui.config('profiling', 'output')

        try:
            if self._output == 'blackbox':
                self._fp = util.stringio()
            elif self._output:
                path = self._ui.expandpath(self._output)
                self._fp = open(path, 'wb')
            else:
                self._fpdoclose = False
                self._fp = self._ui.ferr

            if proffn is not None:
                pass
            elif profiler == 'ls':
                proffn = lsprofile
            elif profiler == 'flame':
                proffn = flameprofile
            else:
                proffn = statprofile

            self._profiler = proffn(self._ui, self._fp)
            self._profiler.__enter__()
        except: # re-raises
            self._closefp()
            raise

    def __exit__(self, exception_type, exception_value, traceback):
        propagate = None
        if self._profiler is not None:
            propagate = self._profiler.__exit__(exception_type, exception_value,
                                                traceback)
            if self._output == 'blackbox':
                val = 'Profile:\n%s' % self._fp.getvalue()
                # ui.log treats the input as a format string,
                # so we need to escape any % signs.
                val = val.replace('%', '%%')
                self._ui.log('profile', val)
        self._closefp()
        return propagate

    def _closefp(self):
        if self._fpdoclose and self._fp is not None:
            self._fp.close()