mercurial/pure/parsers.py

dirstate: avoid use of zip on big lists

In a clean working directory containing 170,000 tracked files, this
improves performance of "hg --time diff" from 1.69 seconds to 1.43.

This idea is due to Siddharth Agarwal.

# parsers.py - Python implementation of parsers.c
#
# Copyright 2009 Matt Mackall <[e-mail]> and others
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from mercurial.node import bin, nullid
from mercurial import util
import struct, zlib

_pack = struct.pack
_unpack = struct.unpack
_compress = zlib.compress
_decompress = zlib.decompress
_sha = util.sha1

def parse_manifest(mfdict, fdict, lines):
    for l in lines.splitlines():
        f, n = l.split('\0')
        if len(n) > 40:
            fdict[f] = n[40:]
            mfdict[f] = bin(n[:40])
        else:
            mfdict[f] = bin(n)

def parse_index2(data, inline):
    def gettype(q):
        return int(q & 0xFFFF)

    def offset_type(offset, type):
        return long(long(offset) << 16 | type)

    indexformatng = ">Qiiiiii20s12x"

    s = struct.calcsize(indexformatng)
    index = []
    cache = None
    off = 0

    l = len(data) - s
    append = index.append
    if inline:
        cache = (0, data)
        while off <= l:
            e = _unpack(indexformatng, data[off:off + s])
            append(e)
            if e[1] < 0:
                break
            off += e[1] + s
    else:
        while off <= l:
            e = _unpack(indexformatng, data[off:off + s])
            append(e)
            off += s

    if off != len(data):
        raise ValueError('corrupt index file')

    if index:
        e = list(index[0])
        type = gettype(e[0])
        e[0] = offset_type(0, type)
        index[0] = tuple(e)

    # add the magic null revision at -1
    index.append((0, 0, 0, -1, -1, -1, -1, nullid))

    return index, cache

def parse_dirstate(dmap, copymap, st):
    parents = [st[:20], st[20: 40]]
    # dereference fields so they will be local in loop
    format = ">cllll"
    e_size = struct.calcsize(format)
    pos1 = 40
    l = len(st)

    # the inner loop
    while pos1 < l:
        pos2 = pos1 + e_size
        e = _unpack(">cllll", st[pos1:pos2]) # a literal here is faster
        pos1 = pos2 + e[4]
        f = st[pos2:pos1]
        if '\0' in f:
            f, c = f.split('\0')
            copymap[f] = c
        dmap[f] = e[:4]
    return parents