contrib/casesmash.py

changegroup: sparsely populate fnodes

Previously, fnodes had a key and empty dict value for every element in
changedfiles. This is somewhat wasteful. Empty dicts in CPython consume
a lot more memory than you would expect - 280 bytes.

On mozilla-central, which has ~190,000 files/fnodes keys, the previous
loop populating fnodes allocated 91,924 KB of memory, most of that for
the empty dicts.

With this patch in place, our peak RSS during mozilla-central clone
drops:

before:  364,356 KB
after:   326,008 KB
delta:   -38,348 KB

When combined with the previous patch, total peak RSS decrease is now
190,116 KB.

import os, __builtin__
from mercurial import util

def lowerwrap(scope, funcname):
    f = getattr(scope, funcname)
    def wrap(fname, *args, **kwargs):
        d, base = os.path.split(fname)
        try:
            files = os.listdir(d or '.')
        except OSError:
            files = []
        if base in files:
            return f(fname, *args, **kwargs)
        for fn in files:
            if fn.lower() == base.lower():
                return f(os.path.join(d, fn), *args, **kwargs)
        return f(fname, *args, **kwargs)
    scope.__dict__[funcname] = wrap

def normcase(path):
    return path.lower()

os.path.normcase = normcase

for f in 'file open'.split():
    lowerwrap(__builtin__, f)

for f in "chmod chown open lstat stat remove unlink".split():
    lowerwrap(os, f)

for f in "exists lexists".split():
    lowerwrap(os.path, f)

lowerwrap(util, 'posixfile')