#!/usr/bin/env python
#
# run-tests.py - Run a set of tests on Mercurial
#
# Copyright 2006 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2, incorporated herein by reference.
# Modifying this script is tricky because it has many modes:
# - serial (default) vs parallel (-jN, N > 1)
# - no coverage (default) vs coverage (-c, -C, -s)
# - temp install (default) vs specific hg script (--with-hg, --local)
# - tests are a mix of shell scripts and Python scripts
#
# If you change this script, it is recommended that you ensure you
# haven't broken it by running it in various modes with a representative
# sample of test scripts. For example:
#
# 1) serial, no coverage, temp install:
# ./run-tests.py test-s*
# 2) serial, no coverage, local hg:
# ./run-tests.py --local test-s*
# 3) serial, coverage, temp install:
# ./run-tests.py -c test-s*
# 4) serial, coverage, local hg:
# ./run-tests.py -c --local test-s* # unsupported
# 5) parallel, no coverage, temp install:
# ./run-tests.py -j2 test-s*
# 6) parallel, no coverage, local hg:
# ./run-tests.py -j2 --local test-s*
# 7) parallel, coverage, temp install:
# ./run-tests.py -j2 -c test-s* # currently broken
# 8) parallel, coverage, local install
# ./run-tests.py -j2 -c --local test-s* # unsupported (and broken)
#
# (You could use any subset of the tests: test-s* happens to match
# enough that it's worth doing parallel runs, few enough that it
# completes fairly quickly, includes both shell and Python scripts, and
# includes some scripts that run daemon processes.)
import difflib
import errno
import optparse
import os
import subprocess
import shutil
import signal
import sys
import tempfile
import time
closefds = os.name == 'posix'
def Popen4(cmd, bufsize=-1):
p = subprocess.Popen(cmd, shell=True, bufsize=bufsize,
close_fds=closefds,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
p.fromchild = p.stdout
p.tochild = p.stdin
p.childerr = p.stderr
return p
# reserved exit code to skip test (used by hghave)
SKIPPED_STATUS = 80
SKIPPED_PREFIX = 'skipped: '
FAILED_PREFIX = 'hghave check failed: '
PYTHON = sys.executable
requiredtools = ["python", "diff", "grep", "unzip", "gunzip", "bunzip2", "sed"]
defaults = {
'jobs': ('HGTEST_JOBS', 1),
'timeout': ('HGTEST_TIMEOUT', 180),
'port': ('HGTEST_PORT', 20059),
}
def parseargs():
parser = optparse.OptionParser("%prog [options] [tests]")
parser.add_option("-C", "--annotate", action="store_true",
help="output files annotated with coverage")
parser.add_option("--child", type="int",
help="run as child process, summary to given fd")
parser.add_option("-c", "--cover", action="store_true",
help="print a test coverage report")
parser.add_option("-f", "--first", action="store_true",
help="exit on the first test failure")
parser.add_option("-i", "--interactive", action="store_true",
help="prompt to accept changed output")
parser.add_option("-j", "--jobs", type="int",
help="number of jobs to run in parallel"
" (default: $%s or %d)" % defaults['jobs'])
parser.add_option("--keep-tmpdir", action="store_true",
help="keep temporary directory after running tests"
" (best used with --tmpdir)")
parser.add_option("-R", "--restart", action="store_true",
help="restart at last error")
parser.add_option("-p", "--port", type="int",
help="port on which servers should listen"
" (default: $%s or %d)" % defaults['port'])
parser.add_option("-r", "--retest", action="store_true",
help="retest failed tests")
parser.add_option("-s", "--cover_stdlib", action="store_true",
help="print a test coverage report inc. standard libraries")
parser.add_option("-t", "--timeout", type="int",
help="kill errant tests after TIMEOUT seconds"
" (default: $%s or %d)" % defaults['timeout'])
parser.add_option("--tmpdir", type="string",
help="run tests in the given temporary directory")
parser.add_option("-v", "--verbose", action="store_true",
help="output verbose messages")
parser.add_option("-n", "--nodiff", action="store_true",
help="skip showing test changes")
parser.add_option("--with-hg", type="string",
metavar="HG",
help="test using specified hg script rather than a "
"temporary installation")
parser.add_option("--local", action="store_true",
help="shortcut for --with-hg=<testdir>/../hg")
parser.add_option("--pure", action="store_true",
help="use pure Python code instead of C extensions")
for option, default in defaults.items():
defaults[option] = int(os.environ.get(*default))
parser.set_defaults(**defaults)
(options, args) = parser.parse_args()
if options.with_hg:
if not (os.path.isfile(options.with_hg) and
os.access(options.with_hg, os.X_OK)):
parser.error('--with-hg must specify an executable hg script')
if not os.path.basename(options.with_hg) == 'hg':
sys.stderr.write('warning: --with-hg should specify an hg script')
if options.local:
testdir = os.path.dirname(os.path.realpath(sys.argv[0]))
hgbin = os.path.join(os.path.dirname(testdir), 'hg')
if not os.access(hgbin, os.X_OK):
parser.error('--local specified, but %r not found or not executable'
% hgbin)
options.with_hg = hgbin
options.anycoverage = (options.cover or
options.cover_stdlib or
options.annotate)
if options.anycoverage and options.with_hg:
# I'm not sure if this is a fundamental limitation or just a
# bug. But I don't want to waste people's time and energy doing
# test runs that don't give the results they want.
parser.error("sorry, coverage options do not work when --with-hg "
"or --local specified")
global vlog
if options.verbose:
if options.jobs > 1 or options.child is not None:
pid = "[%d]" % os.getpid()
else:
pid = None
def vlog(*msg):
if pid:
print pid,
for m in msg:
print m,
print
else:
vlog = lambda *msg: None
if options.jobs < 1:
print >> sys.stderr, 'ERROR: -j/--jobs must be positive'
sys.exit(1)
if options.interactive and options.jobs > 1:
print '(--interactive overrides --jobs)'
options.jobs = 1
return (options, args)
def rename(src, dst):
"""Like os.rename(), trade atomicity and opened files friendliness
for existing destination support.
"""
shutil.copy(src, dst)
os.remove(src)
def splitnewlines(text):
'''like str.splitlines, but only split on newlines.
keep line endings.'''
i = 0
lines = []
while True:
n = text.find('\n', i)
if n == -1:
last = text[i:]
if last:
lines.append(last)
return lines
lines.append(text[i:n+1])
i = n + 1
def parsehghaveoutput(lines):
'''Parse hghave log lines.
Return tuple of lists (missing, failed):
* the missing/unknown features
* the features for which existence check failed'''
missing = []
failed = []
for line in lines:
if line.startswith(SKIPPED_PREFIX):
line = line.splitlines()[0]
missing.append(line[len(SKIPPED_PREFIX):])
elif line.startswith(FAILED_PREFIX):
line = line.splitlines()[0]
failed.append(line[len(FAILED_PREFIX):])
return missing, failed
def showdiff(expected, output):
for line in difflib.unified_diff(expected, output,
"Expected output", "Test output"):
sys.stdout.write(line)
def findprogram(program):
"""Search PATH for a executable program"""
for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
name = os.path.join(p, program)
if os.access(name, os.X_OK):
return name
return None
def checktools():
# Before we go any further, check for pre-requisite tools
# stuff from coreutils (cat, rm, etc) are not tested
for p in requiredtools:
if os.name == 'nt':
p += '.exe'
found = findprogram(p)
if found:
vlog("# Found prerequisite", p, "at", found)
else:
print "WARNING: Did not find prerequisite tool: "+p
def cleanup(options):
if not options.keep_tmpdir:
vlog("# Cleaning up HGTMP", HGTMP)
shutil.rmtree(HGTMP, True)
def usecorrectpython():
# some tests run python interpreter. they must use same
# interpreter we use or bad things will happen.
exedir, exename = os.path.split(sys.executable)
if exename == 'python':
path = findprogram('python')
if os.path.dirname(path) == exedir:
return
vlog('# Making python executable in test path use correct Python')
mypython = os.path.join(BINDIR, 'python')
try:
os.symlink(sys.executable, mypython)
except AttributeError:
# windows fallback
shutil.copyfile(sys.executable, mypython)
shutil.copymode(sys.executable, mypython)
def installhg(options):
vlog("# Performing temporary installation of HG")
installerrs = os.path.join("tests", "install.err")
pure = options.pure and "--pure" or ""
# Run installer in hg root
script = os.path.realpath(sys.argv[0])
hgroot = os.path.dirname(os.path.dirname(script))
os.chdir(hgroot)
cmd = ('%s setup.py %s clean --all'
' install --force --prefix="%s" --install-lib="%s"'
' --install-scripts="%s" >%s 2>&1'
% (sys.executable, pure, INST, PYTHONDIR, BINDIR, installerrs))
vlog("# Running", cmd)
if os.system(cmd) == 0:
if not options.verbose:
os.remove(installerrs)
else:
f = open(installerrs)
for line in f:
print line,
f.close()
sys.exit(1)
os.chdir(TESTDIR)
usecorrectpython()
vlog("# Installing dummy diffstat")
f = open(os.path.join(BINDIR, 'diffstat'), 'w')
f.write('#!' + sys.executable + '\n'
'import sys\n'
'files = 0\n'
'for line in sys.stdin:\n'
' if line.startswith("diff "):\n'
' files += 1\n'
'sys.stdout.write("files patched: %d\\n" % files)\n')
f.close()
os.chmod(os.path.join(BINDIR, 'diffstat'), 0700)
if options.anycoverage:
vlog("# Installing coverage wrapper")
os.environ['COVERAGE_FILE'] = COVERAGE_FILE
if os.path.exists(COVERAGE_FILE):
os.unlink(COVERAGE_FILE)
# Create a wrapper script to invoke hg via coverage.py
os.rename(os.path.join(BINDIR, "hg"), os.path.join(BINDIR, "_hg.py"))
f = open(os.path.join(BINDIR, 'hg'), 'w')
f.write('#!' + sys.executable + '\n')
f.write('import sys, os; os.execv(sys.executable, [sys.executable, '
'"%s", "-x", "-p", "%s"] + sys.argv[1:])\n' %
(os.path.join(TESTDIR, 'coverage.py'),
os.path.join(BINDIR, '_hg.py')))
f.close()
os.chmod(os.path.join(BINDIR, 'hg'), 0700)
def outputcoverage(options):
vlog('# Producing coverage report')
os.chdir(PYTHONDIR)
def covrun(*args):
start = sys.executable, os.path.join(TESTDIR, 'coverage.py')
cmd = '"%s" "%s" %s' % (start[0], start[1], ' '.join(args))
vlog('# Running: %s' % cmd)
os.system(cmd)
omit = [BINDIR, TESTDIR, PYTHONDIR]
if not options.cover_stdlib:
# Exclude as system paths (ignoring empty strings seen on win)
omit += [x for x in sys.path if x != '']
omit = ','.join(omit)
covrun('-c') # combine from parallel processes
for fn in os.listdir(TESTDIR):
if fn.startswith('.coverage.'):
os.unlink(os.path.join(TESTDIR, fn))
covrun('-i', '-r', '"--omit=%s"' % omit) # report
if options.annotate:
adir = os.path.join(TESTDIR, 'annotated')
if not os.path.isdir(adir):
os.mkdir(adir)
covrun('-i', '-a', '"--directory=%s"' % adir, '"--omit=%s"' % omit)
class Timeout(Exception):
pass
def alarmed(signum, frame):
raise Timeout
def run(cmd, options):
"""Run command in a sub-process, capturing the output (stdout and stderr).
Return the exist code, and output."""
# TODO: Use subprocess.Popen if we're running on Python 2.4
if os.name == 'nt' or sys.platform.startswith('java'):
tochild, fromchild = os.popen4(cmd)
tochild.close()
output = fromchild.read()
ret = fromchild.close()
if ret == None:
ret = 0
else:
proc = Popen4(cmd)
try:
output = ''
proc.tochild.close()
output = proc.fromchild.read()
ret = proc.wait()
if os.WIFEXITED(ret):
ret = os.WEXITSTATUS(ret)
except Timeout:
vlog('# Process %d timed out - killing it' % proc.pid)
os.kill(proc.pid, signal.SIGTERM)
ret = proc.wait()
if ret == 0:
ret = signal.SIGTERM << 8
output += ("\n### Abort: timeout after %d seconds.\n"
% options.timeout)
return ret, splitnewlines(output)
def runone(options, test, skips, fails):
'''tristate output:
None -> skipped
True -> passed
False -> failed'''
def skip(msg):
if not options.verbose:
skips.append((test, msg))
else:
print "\nSkipping %s: %s" % (test, msg)
return None
def fail(msg):
fails.append((test, msg))
if not options.nodiff:
print "\nERROR: %s %s" % (test, msg)
return None
vlog("# Test", test)
# create a fresh hgrc
hgrc = file(HGRCPATH, 'w+')
hgrc.write('[ui]\n')
hgrc.write('slash = True\n')
hgrc.write('[defaults]\n')
hgrc.write('backout = -d "0 0"\n')
hgrc.write('commit = -d "0 0"\n')
hgrc.write('tag = -d "0 0"\n')
hgrc.close()
err = os.path.join(TESTDIR, test+".err")
ref = os.path.join(TESTDIR, test+".out")
testpath = os.path.join(TESTDIR, test)
if os.path.exists(err):
os.remove(err) # Remove any previous output files
# Make a tmp subdirectory to work in
tmpd = os.path.join(HGTMP, test)
os.mkdir(tmpd)
os.chdir(tmpd)
try:
tf = open(testpath)
firstline = tf.readline().rstrip()
tf.close()
except:
firstline = ''
lctest = test.lower()
if lctest.endswith('.py') or firstline == '#!/usr/bin/env python':
cmd = '%s "%s"' % (PYTHON, testpath)
elif lctest.endswith('.bat'):
# do not run batch scripts on non-windows
if os.name != 'nt':
return skip("batch script")
# To reliably get the error code from batch files on WinXP,
# the "cmd /c call" prefix is needed. Grrr
cmd = 'cmd /c call "%s"' % testpath
else:
# do not run shell scripts on windows
if os.name == 'nt':
return skip("shell script")
# do not try to run non-executable programs
if not os.path.exists(testpath):
return fail("does not exist")
elif not os.access(testpath, os.X_OK):
return skip("not executable")
cmd = '"%s"' % testpath
if options.timeout > 0:
signal.alarm(options.timeout)
vlog("# Running", cmd)
ret, out = run(cmd, options)
vlog("# Ret was:", ret)
if options.timeout > 0:
signal.alarm(0)
mark = '.'
skipped = (ret == SKIPPED_STATUS)
# If reference output file exists, check test output against it
if os.path.exists(ref):
f = open(ref, "r")
refout = splitnewlines(f.read())
f.close()
else:
refout = []
if skipped:
mark = 's'
missing, failed = parsehghaveoutput(out)
if not missing:
missing = ['irrelevant']
if failed:
fail("hghave failed checking for %s" % failed[-1])
skipped = False
else:
skip(missing[-1])
elif out != refout:
mark = '!'
if ret:
fail("output changed and returned error code %d" % ret)
else:
fail("output changed")
if not options.nodiff:
showdiff(refout, out)
ret = 1
elif ret:
mark = '!'
fail("returned error code %d" % ret)
if not options.verbose:
sys.stdout.write(mark)
sys.stdout.flush()
if ret != 0 and not skipped:
# Save errors to a file for diagnosis
f = open(err, "wb")
for line in out:
f.write(line)
f.close()
# Kill off any leftover daemon processes
try:
fp = file(DAEMON_PIDS)
for line in fp:
try:
pid = int(line)
except ValueError:
continue
try:
os.kill(pid, 0)
vlog('# Killing daemon process %d' % pid)
os.kill(pid, signal.SIGTERM)
time.sleep(0.25)
os.kill(pid, 0)
vlog('# Daemon process %d is stuck - really killing it' % pid)
os.kill(pid, signal.SIGKILL)
except OSError, err:
if err.errno != errno.ESRCH:
raise
fp.close()
os.unlink(DAEMON_PIDS)
except IOError:
pass
os.chdir(TESTDIR)
if not options.keep_tmpdir:
shutil.rmtree(tmpd, True)
if skipped:
return None
return ret == 0
_hgpath = None
def _gethgpath():
"""Return the path to the mercurial package that is actually found by
the current Python interpreter."""
global _hgpath
if _hgpath is not None:
return _hgpath
cmd = '%s -c "import mercurial; print mercurial.__path__[0]"'
pipe = os.popen(cmd % PYTHON)
try:
_hgpath = pipe.read().strip()
finally:
pipe.close()
return _hgpath
def _checkhglib(verb):
"""Ensure that the 'mercurial' package imported by python is
the one we expect it to be. If not, print a warning to stderr."""
expecthg = os.path.join(PYTHONDIR, 'mercurial')
actualhg = _gethgpath()
if actualhg != expecthg:
sys.stderr.write('warning: %s with unexpected mercurial lib: %s\n'
' (expected %s)\n'
% (verb, actualhg, expecthg))
def runchildren(options, tests):
if INST:
installhg(options)
_checkhglib("Testing")
optcopy = dict(options.__dict__)
optcopy['jobs'] = 1
if optcopy['with_hg'] is None:
optcopy['with_hg'] = os.path.join(BINDIR, "hg")
opts = []
for opt, value in optcopy.iteritems():
name = '--' + opt.replace('_', '-')
if value is True:
opts.append(name)
elif value is not None:
opts.append(name + '=' + str(value))
tests.reverse()
jobs = [[] for j in xrange(options.jobs)]
while tests:
for job in jobs:
if not tests: break
job.append(tests.pop())
fps = {}
for j, job in enumerate(jobs):
if not job:
continue
rfd, wfd = os.pipe()
childopts = ['--child=%d' % wfd, '--port=%d' % (options.port + j * 3)]
cmdline = [PYTHON, sys.argv[0]] + opts + childopts + job
vlog(' '.join(cmdline))
fps[os.spawnvp(os.P_NOWAIT, cmdline[0], cmdline)] = os.fdopen(rfd, 'r')
os.close(wfd)
failures = 0
tested, skipped, failed = 0, 0, 0
skips = []
fails = []
while fps:
pid, status = os.wait()
fp = fps.pop(pid)
l = fp.read().splitlines()
test, skip, fail = map(int, l[:3])
split = -fail or len(l)
for s in l[3:split]:
skips.append(s.split(" ", 1))
for s in l[split:]:
fails.append(s.split(" ", 1))
tested += test
skipped += skip
failed += fail
vlog('pid %d exited, status %d' % (pid, status))
failures |= status
print
for s in skips:
print "Skipped %s: %s" % (s[0], s[1])
for s in fails:
print "Failed %s: %s" % (s[0], s[1])
_checkhglib("Tested")
print "# Ran %d tests, %d skipped, %d failed." % (
tested, skipped, failed)
sys.exit(failures != 0)
def runtests(options, tests):
global DAEMON_PIDS, HGRCPATH
DAEMON_PIDS = os.environ["DAEMON_PIDS"] = os.path.join(HGTMP, 'daemon.pids')
HGRCPATH = os.environ["HGRCPATH"] = os.path.join(HGTMP, '.hgrc')
try:
if INST:
installhg(options)
_checkhglib("Testing")
if options.timeout > 0:
try:
signal.signal(signal.SIGALRM, alarmed)
vlog('# Running each test with %d second timeout' %
options.timeout)
except AttributeError:
print 'WARNING: cannot run tests with timeouts'
options.timeout = 0
tested = 0
failed = 0
skipped = 0
if options.restart:
orig = list(tests)
while tests:
if os.path.exists(tests[0] + ".err"):
break
tests.pop(0)
if not tests:
print "running all tests"
tests = orig
skips = []
fails = []
for test in tests:
if options.retest and not os.path.exists(test + ".err"):
skipped += 1
continue
ret = runone(options, test, skips, fails)
if ret is None:
skipped += 1
elif not ret:
if options.interactive:
print "Accept this change? [n] ",
answer = sys.stdin.readline().strip()
if answer.lower() in "y yes".split():
rename(test + ".err", test + ".out")
tested += 1
fails.pop()
continue
failed += 1
if options.first:
break
tested += 1
if options.child:
fp = os.fdopen(options.child, 'w')
fp.write('%d\n%d\n%d\n' % (tested, skipped, failed))
for s in skips:
fp.write("%s %s\n" % s)
for s in fails:
fp.write("%s %s\n" % s)
fp.close()
else:
print
for s in skips:
print "Skipped %s: %s" % s
for s in fails:
print "Failed %s: %s" % s
_checkhglib("Tested")
print "# Ran %d tests, %d skipped, %d failed." % (
tested, skipped, failed)
if options.anycoverage:
outputcoverage(options)
except KeyboardInterrupt:
failed = True
print "\ninterrupted!"
if failed:
sys.exit(1)
def main():
(options, args) = parseargs()
if not options.child:
os.umask(022)
checktools()
# Reset some environment variables to well-known values so that
# the tests produce repeatable output.
os.environ['LANG'] = os.environ['LC_ALL'] = 'C'
os.environ['TZ'] = 'GMT'
os.environ["EMAIL"] = "Foo Bar <foo.bar@example.com>"
os.environ['CDPATH'] = ''
global TESTDIR, HGTMP, INST, BINDIR, PYTHONDIR, COVERAGE_FILE
TESTDIR = os.environ["TESTDIR"] = os.getcwd()
HGTMP = os.environ['HGTMP'] = os.path.realpath(tempfile.mkdtemp('', 'hgtests.',
options.tmpdir))
DAEMON_PIDS = None
HGRCPATH = None
os.environ["HGEDITOR"] = sys.executable + ' -c "import sys; sys.exit(0)"'
os.environ["HGMERGE"] = "internal:merge"
os.environ["HGUSER"] = "test"
os.environ["HGENCODING"] = "ascii"
os.environ["HGENCODINGMODE"] = "strict"
os.environ["HGPORT"] = str(options.port)
os.environ["HGPORT1"] = str(options.port + 1)
os.environ["HGPORT2"] = str(options.port + 2)
if options.with_hg:
INST = None
BINDIR = os.path.dirname(os.path.realpath(options.with_hg))
# This looks redundant with how Python initializes sys.path from
# the location of the script being executed. Needed because the
# "hg" specified by --with-hg is not the only Python script
# executed in the test suite that needs to import 'mercurial'
# ... which means it's not really redundant at all.
PYTHONDIR = BINDIR
else:
INST = os.path.join(HGTMP, "install")
BINDIR = os.environ["BINDIR"] = os.path.join(INST, "bin")
PYTHONDIR = os.path.join(INST, "lib", "python")
os.environ["BINDIR"] = BINDIR
os.environ["PYTHON"] = PYTHON
if not options.child:
path = [BINDIR] + os.environ["PATH"].split(os.pathsep)
os.environ["PATH"] = os.pathsep.join(path)
# Include TESTDIR in PYTHONPATH so that out-of-tree extensions
# can run .../tests/run-tests.py test-foo where test-foo
# adds an extension to HGRC
pypath = [PYTHONDIR, TESTDIR]
# We have to augment PYTHONPATH, rather than simply replacing
# it, in case external libraries are only available via current
# PYTHONPATH. (In particular, the Subversion bindings on OS X
# are in /opt/subversion.)
oldpypath = os.environ.get('PYTHONPATH')
if oldpypath:
pypath.append(oldpypath)
os.environ['PYTHONPATH'] = os.pathsep.join(pypath)
COVERAGE_FILE = os.path.join(TESTDIR, ".coverage")
if len(args) == 0:
args = os.listdir(".")
args.sort()
tests = []
for test in args:
if (test.startswith("test-") and '~' not in test and
('.' not in test or test.endswith('.py') or
test.endswith('.bat'))):
tests.append(test)
if not tests:
print "# Ran 0 tests, 0 skipped, 0 failed."
return
vlog("# Using TESTDIR", TESTDIR)
vlog("# Using HGTMP", HGTMP)
vlog("# Using PATH", os.environ["PATH"])
vlog("# Using PYTHONPATH", os.environ["PYTHONPATH"])
try:
if len(tests) > 1 and options.jobs > 1:
runchildren(options, tests)
else:
runtests(options, tests)
finally:
cleanup(options)
main()