comparison mercurial/store.py @ 41978:a56487081109

store: don't read the whole fncache in memory

In large repositories with lot of files, the fncache grows more than 100 MB and
reading that whole thing into memory slows things down. Let's not read the whole
thing into memory.

This patch changes fncache loading code to read 1 MB at once. Loading 1 MB at
once saves ~1 sec on perffncacheload for our internal repository. I tried
various values such as 0.5 MB, 5 MB, 10 MB but best results were produced using
1 MB as the chunksize.

On a narrow clone with fncache around 40 MB, this patch saves ~0.04 seconds on
average on perffncacheload.

To test the code, I have coded an extension in test-fncache.t which set
chunksize to 1 byte, and the test passes with that.

Differential Revision: https://phab.mercurial-scm.org/D5296

41976:51685c6dcca3

# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

import errno
import hashlib
import os
import stat

from .i18n import _

    util,
    vfs as vfsmod,
)

parsers = policy.importmod(r'parsers')

def _matchtrackedpath(path, matcher):
    """parses a fncache entry and returns whether the entry is tracking a path
    matched by matcher or not.

            fp = self.vfs('fncache', mode='rb')
        except IOError:
            # skip nonexistent file
            self.entries = set()
            return
        self.entries = set(decodedir(fp.read()).splitlines())
        self._checkentries(fp)
        fp.close()

    def _checkentries(self, fp):
        """ make sure there is no empty string in entries """

41978:a56487081109

# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

import errno
import functools
import hashlib
import os
import stat

from .i18n import _

    util,
    vfs as vfsmod,
)

parsers = policy.importmod(r'parsers')
# how much bytes should be read from fncache in one read
# It is done to prevent loading large fncache files into memory
fncache_chunksize = 10 ** 6

def _matchtrackedpath(path, matcher):
    """parses a fncache entry and returns whether the entry is tracking a path
    matched by matcher or not.

            fp = self.vfs('fncache', mode='rb')
        except IOError:
            # skip nonexistent file
            self.entries = set()
            return

        self.entries = set()
        chunk = b''
        for c in iter(functools.partial(fp.read, fncache_chunksize), b''):
            chunk += c
            try:
                p = chunk.rindex(b'\n')
                self.entries.update(decodedir(chunk[:p + 1]).splitlines())
                chunk = chunk[p + 1:]
            except ValueError:
                # substring '\n' not found, maybe the entry is bigger than the
                # chunksize, so let's keep iterating
                pass

        self._checkentries(fp)
        fp.close()

    def _checkentries(self, fp):
        """ make sure there is no empty string in entries """