view doc/check-seclevel.py @ 35776:75bae69747f0

dummysmtpd: don't die on client connection errors

The connection refused error in test-patchbomb-tls.t[1] is sporadic, but one of
the more often seen errors on Windows.  I added enough logging to a file and
dumped it out at the end to make the following observations:

  - The listening socket is successfully created and bound to the port, and the
    "listening at..." message is always logged.

  - Generally, the following is the entire log output, with the "accepted ..."
    message having been added after `sslutil.wrapserversocket`:

          listening at localhost:$HGPORT
          $LOCALIP ssl error
          accepted connect
          accepted connect
          $LOCALIP from=quux to=foo, bar
          $LOCALIP ssl error

  - In the cases that fail, asyncore.loop() in the run() method is exiting, but
    not with an exception.

  - In the cases that fail, the following is logged right after "listening ...":

        Traceback (most recent call last):
          File "c:\\Python27\\lib\\asyncore.py", line 83, in read
            obj.handle_read_event()
          File "c:\\Python27\\lib\\asyncore.py", line 443, in handle_read_event
            self.handle_accept()
          File "../tests/dummysmtpd.py", line 80, in handle_accept
            conn = sslutil.wrapserversocket(conn, ui, certfile=self._certfile)
          File "..\\mercurial\\sslutil.py", line 570, in wrapserversocket
            return sslcontext.wrap_socket(sock, server_side=True)
          File "c:\\Python27\\lib\\ssl.py", line 363, in wrap_socket
            _context=self)
          File "c:\\Python27\\lib\\ssl.py", line 611, in __init__
            self.do_handshake()
          File "c:\\Python27\\lib\\ssl.py", line 840, in do_handshake
            self._sslobj.do_handshake()
        error: [Errno 10054] $ECONNRESET$

  - If the base class handler is overridden completely, the the first "ssl
    error" line is replaced by the stacktrace, but the other lines are
    unchanged.  The client behaves no differently, whether or not the server
    stacktraced.

In general, `./run-tests.py --local -j9 -t9000 test-patchbomb-tls.t
--runs-per-test 20` would show the issue after a run or two.  With this change,
`./run-tests.py --local -j9 -t9000 test-patchbomb-tls.t --loop` ran 800 times
without a hiccup.  This makes me wonder if the other connection refused messages
that bubble up on occasion are caused by a similar issue.  It seems a bit
drastic to kill the whole server on account of a single communication failure
with a client.

# no-check-commit because of handle_error()

[1] https://buildbot.mercurial-scm.org/builders/Win7%20x86_64%20hg%20tests/builds/421/steps/run-tests.py%20%28python%202.7.13%29/logs/stdio

#!/usr/bin/env python
#
# checkseclevel - checking section title levels in each online help document

from __future__ import absolute_import

import optparse
import os
import sys

# import from the live mercurial repo
os.environ['HGMODULEPOLICY'] = 'py'
sys.path.insert(0, "..")
from mercurial import demandimport; demandimport.enable()
from mercurial import (
    commands,
    extensions,
    help,
    minirst,
    ui as uimod,
)

table = commands.table
helptable = help.helptable

level2mark = [b'"', b'=', b'-', b'.', b'#']
reservedmarks = [b'"']

mark2level = {}
for m, l in zip(level2mark, range(len(level2mark))):
    if m not in reservedmarks:
        mark2level[m] = l

initlevel_topic = 0
initlevel_cmd = 1
initlevel_ext = 1
initlevel_ext_cmd = 3

def showavailables(ui, initlevel):
    avail = ('    available marks and order of them in this help: %s\n') % (
        ', '.join(['%r' % (m * 4) for m in level2mark[initlevel + 1:]]))
    ui.warn(avail.encode('utf-8'))

def checkseclevel(ui, doc, name, initlevel):
    ui.note(('checking "%s"\n') % name)
    if not isinstance(doc, bytes):
        doc = doc.encode('utf-8')
    blocks, pruned = minirst.parse(doc, 0, ['verbose'])
    errorcnt = 0
    curlevel = initlevel
    for block in blocks:
        if block[b'type'] != b'section':
            continue
        mark = block[b'underline']
        title = block[b'lines'][0]
        if (mark not in mark2level) or (mark2level[mark] <= initlevel):
            ui.warn((('invalid section mark %r for "%s" of %s\n') %
                     (mark * 4, title, name)).encode('utf-8'))
            showavailables(ui, initlevel)
            errorcnt += 1
            continue
        nextlevel = mark2level[mark]
        if curlevel < nextlevel and curlevel + 1 != nextlevel:
            ui.warn(('gap of section level at "%s" of %s\n') %
                    (title, name))
            showavailables(ui, initlevel)
            errorcnt += 1
            continue
        ui.note(('appropriate section level for "%s %s"\n') %
                (mark * (nextlevel * 2), title))
        curlevel = nextlevel

    return errorcnt

def checkcmdtable(ui, cmdtable, namefmt, initlevel):
    errorcnt = 0
    for k, entry in cmdtable.items():
        name = k.split(b"|")[0].lstrip(b"^")
        if not entry[0].__doc__:
            ui.note(('skip checking %s: no help document\n') %
                    (namefmt % name))
            continue
        errorcnt += checkseclevel(ui, entry[0].__doc__,
                                  namefmt % name,
                                  initlevel)
    return errorcnt

def checkhghelps(ui):
    errorcnt = 0
    for names, sec, doc in helptable:
        if callable(doc):
            doc = doc(ui)
        errorcnt += checkseclevel(ui, doc,
                                  '%s help topic' % names[0],
                                  initlevel_topic)

    errorcnt += checkcmdtable(ui, table, '%s command', initlevel_cmd)

    for name in sorted(list(extensions.enabled()) +
                       list(extensions.disabled())):
        mod = extensions.load(ui, name, None)
        if not mod.__doc__:
            ui.note(('skip checking %s extension: no help document\n') % name)
            continue
        errorcnt += checkseclevel(ui, mod.__doc__,
                                  '%s extension' % name,
                                  initlevel_ext)

        cmdtable = getattr(mod, 'cmdtable', None)
        if cmdtable:
            errorcnt += checkcmdtable(ui, cmdtable,
                                      '%%s command of %s extension' % name,
                                      initlevel_ext_cmd)
    return errorcnt

def checkfile(ui, filename, initlevel):
    if filename == '-':
        filename = 'stdin'
        doc = sys.stdin.read()
    else:
        with open(filename) as fp:
            doc = fp.read()

    ui.note(('checking input from %s with initlevel %d\n') %
            (filename, initlevel))
    return checkseclevel(ui, doc, 'input from %s' % filename, initlevel)

def main():
    optparser = optparse.OptionParser("""%prog [options]

This checks all help documents of Mercurial (topics, commands,
extensions and commands of them), if no file is specified by --file
option.
""")
    optparser.add_option("-v", "--verbose",
                         help="enable additional output",
                         action="store_true")
    optparser.add_option("-d", "--debug",
                         help="debug mode",
                         action="store_true")
    optparser.add_option("-f", "--file",
                         help="filename to read in (or '-' for stdin)",
                         action="store", default="")

    optparser.add_option("-t", "--topic",
                         help="parse file as help topic",
                         action="store_const", dest="initlevel", const=0)
    optparser.add_option("-c", "--command",
                         help="parse file as help of core command",
                         action="store_const", dest="initlevel", const=1)
    optparser.add_option("-e", "--extension",
                         help="parse file as help of extension",
                         action="store_const", dest="initlevel", const=1)
    optparser.add_option("-C", "--extension-command",
                         help="parse file as help of extension command",
                         action="store_const", dest="initlevel", const=3)

    optparser.add_option("-l", "--initlevel",
                         help="set initial section level manually",
                         action="store", type="int", default=0)

    (options, args) = optparser.parse_args()

    ui = uimod.ui.load()
    ui.setconfig('ui', 'verbose', options.verbose, '--verbose')
    ui.setconfig('ui', 'debug', options.debug, '--debug')

    if options.file:
        if checkfile(ui, options.file, options.initlevel):
            sys.exit(1)
    else:
        if checkhghelps(ui):
            sys.exit(1)

if __name__ == "__main__":
    main()