Mercurial > hg
changeset 8326:b7017097a4ec
merge with mpm
| author | Dirkjan Ochtman <dirkjan@ochtman.nl> |
|---|---|
| date | Fri, 08 May 2009 22:35:10 +0200 |
| parents | f2559645643a (current diff) 6b8513f8274a (diff) |
| children | aa25be1c2889 |
| files | |
| diffstat | 1 files changed, 75 insertions(+), 56 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/revlog.py Mon May 04 18:04:41 2009 +0900 +++ b/mercurial/revlog.py Fri May 08 22:35:10 2009 +0200 @@ -31,6 +31,8 @@ REVLOG_DEFAULT_FORMAT = REVLOGNG REVLOG_DEFAULT_VERSION = REVLOG_DEFAULT_FORMAT | REVLOG_DEFAULT_FLAGS +_prereadsize = 1048576 + RevlogError = error.RevlogError LookupError = error.LookupError @@ -315,12 +317,13 @@ def __init__(self): self.size = struct.calcsize(indexformatv0) - def parseindex(self, fp, inline): + def parseindex(self, fp, data, inline): s = self.size index = [] nodemap = {nullid: nullrev} n = off = 0 - data = fp.read() + if len(data) < _prereadsize: + data += fp.read() # read the rest l = len(data) while off + s <= l: cur = data[off:off + s] @@ -358,13 +361,15 @@ def __init__(self): self.size = struct.calcsize(indexformatng) - def parseindex(self, fp, inline): + def parseindex(self, fp, data, inline): try: - size = util.fstat(fp).st_size + size = len(data) + if size == _prereadsize: + size = util.fstat(fp).st_size except AttributeError: size = 0 - if util.openhardlinks() and not inline and size > 1000000: + if util.openhardlinks() and not inline and size > _prereadsize: # big index, let's parse it on demand parser = lazyparser(fp, size) index = lazyindex(parser) @@ -375,7 +380,6 @@ index[0] = e return index, nodemap, None - data = fp.read() # call the C implementation to parse the index data index, nodemap, cache = parsers.parse_index(data, inline) return index, nodemap, cache @@ -422,7 +426,7 @@ self.datafile = indexfile[:-2] + ".d" self.opener = opener self._cache = None - self._chunkcache = None + self._chunkcache = (0, '') self.nodemap = {nullid: nullrev} self.index = [] @@ -432,13 +436,12 @@ if v & REVLOGNG: v |= REVLOGNGINLINEDATA - i = "" + i = '' try: f = self.opener(self.indexfile) - i = f.read(4) - f.seek(0) + i = f.read(_prereadsize) if len(i) > 0: - v = struct.unpack(versionformat, i)[0] + v = struct.unpack(versionformat, i[:4])[0] except IOError, inst: if inst.errno != errno.ENOENT: raise @@ -462,10 +465,12 @@ self._io = revlogoldio() if i: try: - d = self._io.parseindex(f, self._inline) + d = self._io.parseindex(f, i, self._inline) except (ValueError, IndexError), e: raise RevlogError(_("index %s is corrupted") % (self.indexfile)) self.index, self.nodemap, self._chunkcache = d + if not self._chunkcache: + self._chunkcache = (0, '') # add the magic null revision at -1 (if it hasn't been done already) if (self.index == [] or isinstance(self.index, lazyindex) or @@ -907,42 +912,56 @@ p1, p2 = self.parents(node) return hash(text, p1, p2) != node + def _addchunk(self, offset, data): + o, d = self._chunkcache + # try to add to existing cache + if o + len(d) == offset and len(d) + len(data) < _prereadsize: + self._chunkcache = o, d + data + else: + self._chunkcache = offset, data + + def _loadchunk(self, offset, length, df=None): + if not df: + if self._inline: + df = self.opener(self.indexfile) + else: + df = self.opener(self.datafile) + + readahead = max(65536, length) + df.seek(offset) + d = df.read(readahead) + self._addchunk(offset, d) + if readahead > length: + return d[:length] + return d + + def _getchunk(self, offset, length, df=None): + o, d = self._chunkcache + l = len(d) + + # is it in the cache? + cachestart = offset - o + cacheend = cachestart + length + if cachestart >= 0 and cacheend <= l: + if cachestart == 0 and cacheend == l: + return d # avoid a copy + return d[cachestart:cacheend] + + return self._loadchunk(offset, length, df) + + def _prime(self, startrev, endrev, df): + start = self.start(startrev) + end = self.end(endrev) + if self._inline: + start += (startrev + 1) * self._io.size + end += (startrev + 1) * self._io.size + self._loadchunk(start, end - start, df) + def chunk(self, rev, df=None): - def loadcache(df): - if not df: - if self._inline: - df = self.opener(self.indexfile) - else: - df = self.opener(self.datafile) - df.seek(start) - self._chunkcache = (start, df.read(cache_length)) - start, length = self.start(rev), self.length(rev) if self._inline: start += (rev + 1) * self._io.size - end = start + length - - offset = 0 - if not self._chunkcache: - cache_length = max(65536, length) - loadcache(df) - else: - cache_start = self._chunkcache[0] - cache_length = len(self._chunkcache[1]) - cache_end = cache_start + cache_length - if start >= cache_start and end <= cache_end: - # it is cached - offset = start - cache_start - else: - cache_length = max(65536, length) - loadcache(df) - - # avoid copying large chunks - c = self._chunkcache[1] - if cache_length != length: - c = c[offset:offset + length] - - return decompress(c) + return decompress(self._getchunk(start, length, df)) def revdiff(self, rev1, rev2): """return or calculate a delta between two revisions""" @@ -978,10 +997,12 @@ self._loadindex(base, rev + 1) if not self._inline and rev > base + 1: df = self.opener(self.datafile) + self._prime(base, rev, df) else: self._loadindex(base, rev + 1) if not self._inline and rev > base: df = self.opener(self.datafile) + self._prime(base, rev, df) text = self.chunk(base, df=df) bins = [self.chunk(r, df) for r in xrange(base + 1, rev + 1)] @@ -995,14 +1016,9 @@ return text def checkinlinesize(self, tr, fp=None): - if not self._inline: + if not self._inline or (self.start(-2) + self.length(-2)) < 131072: return - if not fp: - fp = self.opener(self.indexfile, 'r') - fp.seek(0, 2) - size = fp.tell() - if size < 131072: - return + trinfo = tr.find(self.indexfile) if trinfo == None: raise RevlogError(_("%s not found in the transaction") @@ -1012,19 +1028,22 @@ dataoff = self.start(trindex) tr.add(self.datafile, dataoff) + + if fp: + fp.flush() + fp.close() + df = self.opener(self.datafile, 'w') try: calc = self._io.size for r in self: start = self.start(r) + (r + 1) * calc length = self.length(r) - fp.seek(start) - d = fp.read(length) + d = self._getchunk(start, length) df.write(d) finally: df.close() - fp.close() fp = self.opener(self.indexfile, 'w', atomictemp=True) self.version &= ~(REVLOGNGINLINEDATA) self._inline = False @@ -1037,7 +1056,7 @@ fp.rename() tr.replace(self.indexfile, trindex * calc) - self._chunkcache = None + self._chunkcache = (0, '') def addrevision(self, text, transaction, link, p1, p2, d=None): """add a revision to the log @@ -1322,7 +1341,7 @@ # then reset internal state in memory to forget those revisions self._cache = None - self._chunkcache = None + self._chunkcache = (0, '') for x in xrange(rev, len(self)): del self.nodemap[self.node(x)]