findrenames: Optimise "addremove -s100" by matching files by their SHA1 hashes.
We speed up 'findrenames' for the usecase when a user specifies they
want a similarity of 100% by matching files by their exact SHA1 hash
value. This reduces the number of comparisons required to find exact
matches from O(n^2) to O(n).
While it would be nice if we could just use mercurial's pre-calculated
SHA1 hash for existing files, this hash includes the file's ancestor
information making it unsuitable for our purposes. Instead, we calculate
the hash of old content from scratch.
The following benchmarks were taken on the current head of crew:
addremove 100% similarity:
rm -rf *; hg up -C; mv tests tests.new
hg --time addremove -s100 --dry-run
before: real 176.350 secs (user 128.890+0.000 sys 47.430+0.000)
after: real 2.130 secs (user 1.890+0.000 sys 0.240+0.000)
addremove 75% similarity:
rm -rf *; hg up -C; mv tests tests.new; \
for i in tests.new/*; do echo x >> $i; done
hg --time addremove -s75 --dry-run
before: real 264.560 secs (user 215.130+0.000 sys 49.410+0.000)
after: real 218.710 secs (user 172.790+0.000 sys 45.870+0.000)
#!/usr/bin/env python
"""Test the running system for features availability. Exit with zero
if all features are there, non-zero otherwise. If a feature name is
prefixed with "no-", the absence of feature is tested.
"""
import optparse
import os
import re
import sys
import tempfile
tempprefix = 'hg-hghave-'
def matchoutput(cmd, regexp, ignorestatus=False):
"""Return True if cmd executes successfully and its output
is matched by the supplied regular expression.
"""
r = re.compile(regexp)
fh = os.popen(cmd)
s = fh.read()
try:
ret = fh.close()
except IOError:
# Happen in Windows test environment
ret = 1
return (ignorestatus or ret is None) and r.search(s)
def has_baz():
return matchoutput('baz --version 2>&1', r'baz Bazaar version')
def has_bzr():
try:
import bzrlib
return bzrlib.__doc__ != None
except ImportError:
return False
def has_bzr114():
try:
import bzrlib
return (bzrlib.__doc__ != None
and bzrlib.version_info[:2] >= (1, 14))
except ImportError:
return False
def has_cvs():
re = r'Concurrent Versions System.*?server'
return matchoutput('cvs --version 2>&1', re)
def has_darcs():
return matchoutput('darcs --version', r'2\.[2-9]', True)
def has_mtn():
return matchoutput('mtn --version', r'monotone', True) and not matchoutput(
'mtn --version', r'monotone 0\.(\d|[12]\d|3[01])[^\d]', True)
def has_eol_in_paths():
try:
fd, path = tempfile.mkstemp(prefix=tempprefix, suffix='\n\r')
os.close(fd)
os.remove(path)
return True
except:
return False
def has_executablebit():
fd, path = tempfile.mkstemp(prefix=tempprefix)
os.close(fd)
try:
s = os.lstat(path).st_mode
os.chmod(path, s | 0100)
return (os.lstat(path).st_mode & 0100 != 0)
finally:
os.remove(path)
def has_icasefs():
# Stolen from mercurial.util
fd, path = tempfile.mkstemp(prefix=tempprefix, dir='.')
os.close(fd)
try:
s1 = os.stat(path)
d, b = os.path.split(path)
p2 = os.path.join(d, b.upper())
if path == p2:
p2 = os.path.join(d, b.lower())
try:
s2 = os.stat(p2)
return s2 == s1
except:
return False
finally:
os.remove(path)
def has_inotify():
try:
import hgext.inotify.linux.watcher
return True
except ImportError:
return False
def has_fifo():
return hasattr(os, "mkfifo")
def has_hotshot():
try:
# hotshot.stats tests hotshot and many problematic dependencies
# like profile.
import hotshot.stats
return True
except ImportError:
return False
def has_lsprof():
try:
import _lsprof
return True
except ImportError:
return False
def has_git():
return matchoutput('git --version 2>&1', r'^git version')
def has_docutils():
try:
from docutils.core import publish_cmdline
return True
except ImportError:
return False
def has_svn():
return matchoutput('svn --version 2>&1', r'^svn, version') and \
matchoutput('svnadmin --version 2>&1', r'^svnadmin, version')
def has_svn_bindings():
try:
import svn.core
version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR
if version < (1, 4):
return False
return True
except ImportError:
return False
def has_p4():
return matchoutput('p4 -V', r'Rev\. P4/') and matchoutput('p4d -V', r'Rev\. P4D/')
def has_symlink():
return hasattr(os, "symlink")
def has_tla():
return matchoutput('tla --version 2>&1', r'The GNU Arch Revision')
def has_gpg():
return matchoutput('gpg --version 2>&1', r'GnuPG')
def has_unix_permissions():
d = tempfile.mkdtemp(prefix=tempprefix, dir=".")
try:
fname = os.path.join(d, 'foo')
for umask in (077, 007, 022):
os.umask(umask)
f = open(fname, 'w')
f.close()
mode = os.stat(fname).st_mode
os.unlink(fname)
if mode & 0777 != ~umask & 0666:
return False
return True
finally:
os.rmdir(d)
def has_pygments():
try:
import pygments
return True
except ImportError:
return False
def has_outer_repo():
return matchoutput('hg root 2>&1', r'')
checks = {
"baz": (has_baz, "GNU Arch baz client"),
"bzr": (has_bzr, "Canonical's Bazaar client"),
"bzr114": (has_bzr114, "Canonical's Bazaar client >= 1.14"),
"cvs": (has_cvs, "cvs client/server"),
"darcs": (has_darcs, "darcs client"),
"eol-in-paths": (has_eol_in_paths, "end-of-lines in paths"),
"execbit": (has_executablebit, "executable bit"),
"fifo": (has_fifo, "named pipes"),
"git": (has_git, "git command line client"),
"gpg": (has_gpg, "gpg client"),
"hotshot": (has_hotshot, "python hotshot module"),
"icasefs": (has_icasefs, "case insensitive file system"),
"inotify": (has_inotify, "inotify extension support"),
"lsprof": (has_lsprof, "python lsprof module"),
"mtn": (has_mtn, "monotone client (> 0.31)"),
"outer-repo": (has_outer_repo, "outer repo"),
"p4": (has_p4, "Perforce server and client"),
"pygments": (has_pygments, "Pygments source highlighting library"),
"docutils": (has_docutils, "Docutils text processing library"),
"svn": (has_svn, "subversion client and admin tools"),
"svn-bindings": (has_svn_bindings, "subversion python bindings"),
"symlink": (has_symlink, "symbolic links"),
"tla": (has_tla, "GNU Arch tla client"),
"unix-permissions": (has_unix_permissions, "unix-style permissions"),
}
def list_features():
for name, feature in checks.iteritems():
desc = feature[1]
print name + ':', desc
def test_features():
failed = 0
for name, feature in checks.iteritems():
check, _ = feature
try:
check()
except Exception, e:
print "feature %s failed: %s" % (name, e)
failed += 1
return failed
parser = optparse.OptionParser("%prog [options] [features]")
parser.add_option("--test-features", action="store_true",
help="test available features")
parser.add_option("--list-features", action="store_true",
help="list available features")
parser.add_option("-q", "--quiet", action="store_true",
help="check features silently")
if __name__ == '__main__':
options, args = parser.parse_args()
if options.list_features:
list_features()
sys.exit(0)
if options.test_features:
sys.exit(test_features())
quiet = options.quiet
failures = 0
def error(msg):
global failures
if not quiet:
sys.stderr.write(msg + '\n')
failures += 1
for feature in args:
negate = feature.startswith('no-')
if negate:
feature = feature[3:]
if feature not in checks:
error('skipped: unknown feature: ' + feature)
continue
check, desc = checks[feature]
try:
available = check()
except Exception, e:
error('hghave check failed: ' + feature)
continue
if not negate and not available:
error('skipped: missing feature: ' + desc)
elif negate and available:
error('skipped: system supports %s' % desc)
if failures != 0:
sys.exit(1)