Mercurial > hg
comparison tests/run-tests.py @ 35190:bd8875b6473c
run-tests: mechanism to report exceptions during test execution
Sometimes when running tests you introduce a ton of exceptions.
The most extreme example of this is running Mercurial with Python 3,
which currently spews thousands of exceptions when running the test
harness.
This commit adds an opt-in feature to run-tests.py to aggregate
exceptions encountered by `hg` when running tests.
When --exceptions is used, the test harness enables the
"logexceptions" extension in the test environment. This extension
wraps the Mercurial function to handle exceptions and writes
information about the exception to a random filename in a directory
defined by the test harness via an environment variable. At the
end of the test harness, these files are parsed, aggregated, and
a list of all unique Mercurial frames triggering exceptions is
printed in order of frequency.
This feature is intended to aid Python 3 development. I've only
really tested it on Python 3. There is no shortage of improvements
that could be made. e.g. we could write a separate file containing
the exception report - maybe even an HTML report. We also don't
capture which tests demonstrate the exceptions, so there's no turnkey
way to test whether a code change made an exception disappear.
Perfect is the enemy of good. I think the current patch is useful
enough to land. Whoever uses it can send patches to imprve its
usefulness.
Differential Revision: https://phab.mercurial-scm.org/D1477
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Mon, 20 Nov 2017 23:02:32 -0800 |
parents | 073843b5e353 |
children | f12747de13ea |
comparison
equal
deleted
inserted
replaced
35189:073843b5e353 | 35190:bd8875b6473c |
---|---|
44 # includes some scripts that run daemon processes.) | 44 # includes some scripts that run daemon processes.) |
45 | 45 |
46 from __future__ import absolute_import, print_function | 46 from __future__ import absolute_import, print_function |
47 | 47 |
48 import argparse | 48 import argparse |
49 import collections | |
49 import difflib | 50 import difflib |
50 import distutils.version as version | 51 import distutils.version as version |
51 import errno | 52 import errno |
52 import json | 53 import json |
53 import os | 54 import os |
371 hgconf = parser.add_argument_group('Mercurial Configuration') | 372 hgconf = parser.add_argument_group('Mercurial Configuration') |
372 hgconf.add_argument("--chg", action="store_true", | 373 hgconf.add_argument("--chg", action="store_true", |
373 help="install and use chg wrapper in place of hg") | 374 help="install and use chg wrapper in place of hg") |
374 hgconf.add_argument("--compiler", | 375 hgconf.add_argument("--compiler", |
375 help="compiler to build with") | 376 help="compiler to build with") |
376 hgconf.add_argument('--extra-config-opt', action="append", | 377 hgconf.add_argument('--extra-config-opt', action="append", default=[], |
377 help='set the given config opt in the test hgrc') | 378 help='set the given config opt in the test hgrc') |
378 hgconf.add_argument("-l", "--local", action="store_true", | 379 hgconf.add_argument("-l", "--local", action="store_true", |
379 help="shortcut for --with-hg=<testdir>/../hg, " | 380 help="shortcut for --with-hg=<testdir>/../hg, " |
380 "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set") | 381 "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set") |
381 hgconf.add_argument("--ipv6", action="store_true", | 382 hgconf.add_argument("--ipv6", action="store_true", |
402 reporting.add_argument("--color", choices=["always", "auto", "never"], | 403 reporting.add_argument("--color", choices=["always", "auto", "never"], |
403 default=os.environ.get('HGRUNTESTSCOLOR', 'auto'), | 404 default=os.environ.get('HGRUNTESTSCOLOR', 'auto'), |
404 help="colorisation: always|auto|never (default: auto)") | 405 help="colorisation: always|auto|never (default: auto)") |
405 reporting.add_argument("-c", "--cover", action="store_true", | 406 reporting.add_argument("-c", "--cover", action="store_true", |
406 help="print a test coverage report") | 407 help="print a test coverage report") |
408 reporting.add_argument('--exceptions', action='store_true', | |
409 help='log all exceptions and generate an exception report') | |
407 reporting.add_argument("-H", "--htmlcov", action="store_true", | 410 reporting.add_argument("-H", "--htmlcov", action="store_true", |
408 help="create an HTML report of the coverage of the files") | 411 help="create an HTML report of the coverage of the files") |
409 reporting.add_argument("--json", action="store_true", | 412 reporting.add_argument("--json", action="store_true", |
410 help="store test result data in 'report.json' file") | 413 help="store test result data in 'report.json' file") |
411 reporting.add_argument("--outputdir", | 414 reporting.add_argument("--outputdir", |
2113 if failed: | 2116 if failed: |
2114 self.stream.writeln('python hash seed: %s' % | 2117 self.stream.writeln('python hash seed: %s' % |
2115 os.environ['PYTHONHASHSEED']) | 2118 os.environ['PYTHONHASHSEED']) |
2116 if self._runner.options.time: | 2119 if self._runner.options.time: |
2117 self.printtimes(result.times) | 2120 self.printtimes(result.times) |
2121 | |
2122 if self._runner.options.exceptions: | |
2123 exceptions = aggregateexceptions( | |
2124 os.path.join(self._runner._outputdir, b'exceptions')) | |
2125 total = sum(exceptions.values()) | |
2126 | |
2127 self.stream.writeln('Exceptions Report:') | |
2128 self.stream.writeln('%d total from %d frames' % | |
2129 (total, len(exceptions))) | |
2130 for (frame, line, exc), count in exceptions.most_common(): | |
2131 self.stream.writeln('%d\t%s: %s' % (count, frame, exc)) | |
2132 | |
2118 self.stream.flush() | 2133 self.stream.flush() |
2119 | 2134 |
2120 return result | 2135 return result |
2121 | 2136 |
2122 def _bisecttests(self, tests): | 2137 def _bisecttests(self, tests): |
2499 elif 'HGTEST_SLOW' in os.environ: | 2514 elif 'HGTEST_SLOW' in os.environ: |
2500 del os.environ['HGTEST_SLOW'] | 2515 del os.environ['HGTEST_SLOW'] |
2501 | 2516 |
2502 self._coveragefile = os.path.join(self._testdir, b'.coverage') | 2517 self._coveragefile = os.path.join(self._testdir, b'.coverage') |
2503 | 2518 |
2519 if self.options.exceptions: | |
2520 exceptionsdir = os.path.join(self._outputdir, b'exceptions') | |
2521 try: | |
2522 os.makedirs(exceptionsdir) | |
2523 except OSError as e: | |
2524 if e.errno != errno.EEXIST: | |
2525 raise | |
2526 | |
2527 # Remove all existing exception reports. | |
2528 for f in os.listdir(exceptionsdir): | |
2529 os.unlink(os.path.join(exceptionsdir, f)) | |
2530 | |
2531 osenvironb[b'HGEXCEPTIONSDIR'] = exceptionsdir | |
2532 logexceptions = os.path.join(self._testdir, b'logexceptions.py') | |
2533 self.options.extra_config_opt.append( | |
2534 'extensions.logexceptions=%s' % logexceptions.decode('utf-8')) | |
2535 | |
2504 vlog("# Using TESTDIR", self._testdir) | 2536 vlog("# Using TESTDIR", self._testdir) |
2505 vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR']) | 2537 vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR']) |
2506 vlog("# Using HGTMP", self._hgtmp) | 2538 vlog("# Using HGTMP", self._hgtmp) |
2507 vlog("# Using PATH", os.environ["PATH"]) | 2539 vlog("# Using PATH", os.environ["PATH"]) |
2508 vlog("# Using", IMPL_PATH, osenvironb[IMPL_PATH]) | 2540 vlog("# Using", IMPL_PATH, osenvironb[IMPL_PATH]) |
2951 vlog("# Found prerequisite", p, "at", found) | 2983 vlog("# Found prerequisite", p, "at", found) |
2952 else: | 2984 else: |
2953 print("WARNING: Did not find prerequisite tool: %s " % | 2985 print("WARNING: Did not find prerequisite tool: %s " % |
2954 p.decode("utf-8")) | 2986 p.decode("utf-8")) |
2955 | 2987 |
2988 def aggregateexceptions(path): | |
2989 exceptions = collections.Counter() | |
2990 | |
2991 for f in os.listdir(path): | |
2992 with open(os.path.join(path, f), 'rb') as fh: | |
2993 data = fh.read().split(b'\0') | |
2994 if len(data) != 4: | |
2995 continue | |
2996 | |
2997 exc, mainframe, hgframe, hgline = data | |
2998 exc = exc.decode('utf-8') | |
2999 mainframe = mainframe.decode('utf-8') | |
3000 hgframe = hgframe.decode('utf-8') | |
3001 hgline = hgline.decode('utf-8') | |
3002 exceptions[(hgframe, hgline, exc)] += 1 | |
3003 | |
3004 return exceptions | |
3005 | |
2956 if __name__ == '__main__': | 3006 if __name__ == '__main__': |
2957 runner = TestRunner() | 3007 runner = TestRunner() |
2958 | 3008 |
2959 try: | 3009 try: |
2960 import msvcrt | 3010 import msvcrt |