worker: rewrite error handling so os._exit covers all cases
Previously the worker error handling is like:
pid = os.fork() --+
if pid == 0: |
.... | problematic
.... --+
try: --+
.... | worker error handling
--+
If a signal arrives when Python is executing the "problematic" lines, an
external error handling (dispatch.py) will take over the control flow and
it's no longer guaranteed "os._exit" is called (see
86cd09bc13ba for why it
is necessary).
This patch rewrites the error handling so it covers all possible code paths
for a worker even during fork.
Note: "os.getpid() == parentpid" is used to test if the process is parent or
not intentionally, instead of checking "pid", because "pid = os.fork()" may
be not atomic - it's possible that that a signal hits the worker before the
assignment completes [1]. The newly added test replaces "os.fork" to
exercise that extreme case.
[1]: CPython compiles "pid = os.fork()" to 2 byte codes: "CALL_FUNCTION" and
"STORE_FAST", so it's probably not atomic:
def f():
pid = os.fork()
dis.dis(f)
2 0 LOAD_GLOBAL 0 (os)
3 LOAD_ATTR 1 (fork)
6 CALL_FUNCTION 0
9 STORE_FAST 0 (pid)
12 LOAD_CONST 0 (None)
15 RETURN_VALUE
# profiling.py - profiling functions
#
# Copyright 2016 Gregory Szorc <gregory.szorc@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import, print_function
import contextlib
from .i18n import _
from . import (
encoding,
error,
util,
)
@contextlib.contextmanager
def lsprofile(ui, fp):
format = ui.config('profiling', 'format', default='text')
field = ui.config('profiling', 'sort', default='inlinetime')
limit = ui.configint('profiling', 'limit', default=30)
climit = ui.configint('profiling', 'nested', default=0)
if format not in ['text', 'kcachegrind']:
ui.warn(_("unrecognized profiling format '%s'"
" - Ignored\n") % format)
format = 'text'
try:
from . import lsprof
except ImportError:
raise error.Abort(_(
'lsprof not available - install from '
'http://codespeak.net/svn/user/arigo/hack/misc/lsprof/'))
p = lsprof.Profiler()
p.enable(subcalls=True)
try:
yield
finally:
p.disable()
if format == 'kcachegrind':
from . import lsprofcalltree
calltree = lsprofcalltree.KCacheGrind(p)
calltree.output(fp)
else:
# format == 'text'
stats = lsprof.Stats(p.getstats())
stats.sort(field)
stats.pprint(limit=limit, file=fp, climit=climit)
@contextlib.contextmanager
def flameprofile(ui, fp):
try:
from flamegraph import flamegraph
except ImportError:
raise error.Abort(_(
'flamegraph not available - install from '
'https://github.com/evanhempel/python-flamegraph'))
# developer config: profiling.freq
freq = ui.configint('profiling', 'freq', default=1000)
filter_ = None
collapse_recursion = True
thread = flamegraph.ProfileThread(fp, 1.0 / freq,
filter_, collapse_recursion)
start_time = util.timer()
try:
thread.start()
yield
finally:
thread.stop()
thread.join()
print('Collected %d stack frames (%d unique) in %2.2f seconds.' % (
util.timer() - start_time, thread.num_frames(),
thread.num_frames(unique=True)))
@contextlib.contextmanager
def statprofile(ui, fp):
from . import statprof
freq = ui.configint('profiling', 'freq', default=1000)
if freq > 0:
# Cannot reset when profiler is already active. So silently no-op.
if statprof.state.profile_level == 0:
statprof.reset(freq)
else:
ui.warn(_("invalid sampling frequency '%s' - ignoring\n") % freq)
statprof.start(mechanism='thread')
try:
yield
finally:
data = statprof.stop()
profformat = ui.config('profiling', 'statformat', 'hotpath')
formats = {
'byline': statprof.DisplayFormats.ByLine,
'bymethod': statprof.DisplayFormats.ByMethod,
'hotpath': statprof.DisplayFormats.Hotpath,
'json': statprof.DisplayFormats.Json,
'chrome': statprof.DisplayFormats.Chrome,
}
if profformat in formats:
displayformat = formats[profformat]
else:
ui.warn(_('unknown profiler output format: %s\n') % profformat)
displayformat = statprof.DisplayFormats.Hotpath
kwargs = {}
def fraction(s):
if s.endswith('%'):
v = float(s[:-1]) / 100
else:
v = float(s)
if 0 <= v <= 1:
return v
raise ValueError(s)
if profformat == 'chrome':
showmin = ui.configwith(fraction, 'profiling', 'showmin', 0.005)
showmax = ui.configwith(fraction, 'profiling', 'showmax', 0.999)
kwargs.update(minthreshold=showmin, maxthreshold=showmax)
statprof.display(fp, data=data, format=displayformat, **kwargs)
@contextlib.contextmanager
def profile(ui):
"""Start profiling.
Profiling is active when the context manager is active. When the context
manager exits, profiling results will be written to the configured output.
"""
profiler = encoding.environ.get('HGPROF')
if profiler is None:
profiler = ui.config('profiling', 'type', default='stat')
if profiler not in ('ls', 'stat', 'flame'):
ui.warn(_("unrecognized profiler '%s' - ignored\n") % profiler)
profiler = 'stat'
output = ui.config('profiling', 'output')
if output == 'blackbox':
fp = util.stringio()
elif output:
path = ui.expandpath(output)
fp = open(path, 'wb')
else:
fp = ui.ferr
try:
if profiler == 'ls':
proffn = lsprofile
elif profiler == 'flame':
proffn = flameprofile
else:
proffn = statprofile
with proffn(ui, fp):
yield
finally:
if output:
if output == 'blackbox':
val = 'Profile:\n%s' % fp.getvalue()
# ui.log treats the input as a format string,
# so we need to escape any % signs.
val = val.replace('%', '%%')
ui.log('profile', val)
fp.close()
@contextlib.contextmanager
def maybeprofile(ui):
"""Profile if enabled, else do nothing.
This context manager can be used to optionally profile if profiling
is enabled. Otherwise, it does nothing.
The purpose of this context manager is to make calling code simpler:
just use a single code path for calling into code you may want to profile
and this function determines whether to start profiling.
"""
if ui.configbool('profiling', 'enabled'):
with profile(ui):
yield
else:
yield