Mercurial > hg
view mercurial/statichttprepo.py @ 21424:d13b4ecdb680
test: split test-largefile.t in multiple file
The `test-largefiles.t` unified test is significantly longer (about 30%) than
any other tests in the mercurial test suite. As a result, its is alway the last
test my test runner is waiting for at the end of a run.
In practice, this means that `test-largefile.t` is wasting half a minute of my
life every times I'm running the mercurial test suites. This probably mean more
a few cumulated day by now.
I've finally decided to split it up in multiple smaller tests to bring it back in
reasonable length.
This changeset extracts independent test cases in two files. One dedicated to
wire protocole testing, and another one dedicated to all other tests that could
be independently extracted.
No test case were haltered in the making of this changeset.
Various timing available below. All timing have been done on a with 90 jobs on a
64 cores machine. Similar result are shown on firefly (20 jobs on 12 core).
General timing of the whole run
--------------------------------
We see a 25% real time improvement for no significant cpu time impact.
Before split:
real 2m1.149s
user 58m4.662s
sys 11m28.563s
After split:
real 1m31.977s
user 57m45.993s
sys 11m33.634s
Last test to finish (using run-test.py --time)
----------------------------------------------
test-largefile.t is now finishing at the same time than other slow tests.
Before split:
Time Test
119.280 test-largefiles.t
93.995 test-mq.t
89.897 test-subrepo.t
86.920 test-glog.t
85.508 test-rename-merge2.t
83.594 test-revset.t
79.824 test-keyword.t
78.077 test-mq-header-date.t
After split:
Time Test
90.414 test-mq.t
88.594 test-largefiles.t
85.363 test-subrepo.t
81.059 test-glog.t
78.927 test-rename-merge2.t
78.021 test-revset.t
77.777 test-command-template.t
Timing of largefile test themself
-----------------------------------
Running only tests prefixed with "test-largefiles".
No significant change in cumulated time.
Before:
Time Test
58.673 test-largefiles.t
2.931 test-largefiles-cache.t
0.583 test-largefiles-small-disk.t
After:
Time Test
31.754 test-largefiles.t
17.460 test-largefiles-misc.t
8.888 test-largefiles-wireproto.t
2.864 test-largefiles-cache.t
0.580 test-largefiles-small-disk.t
author | Pierre-Yves David <pierre-yves.david@fb.com> |
---|---|
date | Fri, 16 May 2014 13:18:57 -0700 |
parents | af4158b8876b |
children | 72319005f5fb |
line wrap: on
line source
# statichttprepo.py - simple http repository class for mercurial # # This provides read-only repo access to repositories exported via static http # # Copyright 2005-2007 Matt Mackall <mpm@selenic.com> # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. from i18n import _ import changelog, byterange, url, error import localrepo, manifest, util, scmutil, store import urllib, urllib2, errno, os class httprangereader(object): def __init__(self, url, opener): # we assume opener has HTTPRangeHandler self.url = url self.pos = 0 self.opener = opener self.name = url def seek(self, pos): self.pos = pos def read(self, bytes=None): req = urllib2.Request(self.url) end = '' if bytes: end = self.pos + bytes - 1 if self.pos or end: req.add_header('Range', 'bytes=%d-%s' % (self.pos, end)) try: f = self.opener.open(req) data = f.read() # Python 2.6+ defines a getcode() function, and 2.4 and # 2.5 appear to always have an undocumented code attribute # set. If we can't read either of those, fall back to 206 # and hope for the best. code = getattr(f, 'getcode', lambda : getattr(f, 'code', 206))() except urllib2.HTTPError, inst: num = inst.code == 404 and errno.ENOENT or None raise IOError(num, inst) except urllib2.URLError, inst: raise IOError(None, inst.reason[1]) if code == 200: # HTTPRangeHandler does nothing if remote does not support # Range headers and returns the full entity. Let's slice it. if bytes: data = data[self.pos:self.pos + bytes] else: data = data[self.pos:] elif bytes: data = data[:bytes] self.pos += len(data) return data def readlines(self): return self.read().splitlines(True) def __iter__(self): return iter(self.readlines()) def close(self): pass def build_opener(ui, authinfo): # urllib cannot handle URLs with embedded user or passwd urlopener = url.opener(ui, authinfo) urlopener.add_handler(byterange.HTTPRangeHandler()) class statichttpvfs(scmutil.abstractvfs): def __init__(self, base): self.base = base def __call__(self, path, mode="r", atomictemp=None): if mode not in ('r', 'rb'): raise IOError('Permission denied') f = "/".join((self.base, urllib.quote(path))) return httprangereader(f, urlopener) def join(self, path): if path: return os.path.join(self.base, path) else: return self.base return statichttpvfs class statichttppeer(localrepo.localpeer): def local(self): return None def canpush(self): return False class statichttprepository(localrepo.localrepository): supported = localrepo.localrepository._basesupported def __init__(self, ui, path): self._url = path self.ui = ui self.root = path u = util.url(path.rstrip('/') + "/.hg") self.path, authinfo = u.authinfo() opener = build_opener(ui, authinfo) self.opener = opener(self.path) self.vfs = self.opener self._phasedefaults = [] try: requirements = scmutil.readrequires(self.opener, self.supported) except IOError, inst: if inst.errno != errno.ENOENT: raise requirements = set() # check if it is a non-empty old-style repository try: fp = self.opener("00changelog.i") fp.read(1) fp.close() except IOError, inst: if inst.errno != errno.ENOENT: raise # we do not care about empty old-style repositories here msg = _("'%s' does not appear to be an hg repository") % path raise error.RepoError(msg) # setup store self.store = store.store(requirements, self.path, opener) self.spath = self.store.path self.sopener = self.store.opener self.svfs = self.sopener self.sjoin = self.store.join self._filecache = {} self.requirements = requirements self.manifest = manifest.manifest(self.sopener) self.changelog = changelog.changelog(self.sopener) self._tags = None self.nodetagscache = None self._branchcaches = {} self.encodepats = None self.decodepats = None def _restrictcapabilities(self, caps): caps = super(statichttprepository, self)._restrictcapabilities(caps) return caps.difference(["pushkey"]) def url(self): return self._url def local(self): return False def peer(self): return statichttppeer(self) def lock(self, wait=True): raise util.Abort(_('cannot lock static-http repository')) def instance(ui, path, create): if create: raise util.Abort(_('cannot create new static-http repository')) return statichttprepository(ui, path[7:])