comparison hgext/fsmonitor/__init__.py @ 31846:1064a296a2a7

fsmonitor: match watchman and filesystem encoding

watchman's paths encoding can differ from filesystem encoding. For example,
on Windows, it's always utf-8.

Before this patch, on Windows, mismatch in path comparison between fsmonitor
state and osutil.statfiles would yield a clean status for added/modified files.

In addition to status reporting wrong results, this leads to files being
discarded from changesets while doing history editing operations such as rebase.

Benchmark:

There is a little overhead at module import:
python -m timeit "import hgext.fsmonitor"
Windows before patch: 1000000 loops, best of 3: 0.563 usec per loop
Windows after patch: 1000000 loops, best of 3: 0.583 usec per loop
Linx before patch: 1000000 loops, best of 3: 0.579 usec per loop
Linux after patch: 1000000 loops, best of 3: 0.588 usec per loop

10000 calls to _watchmantofsencoding:
python -m timeit -s "from hgext.fsmonitor import _watchmantofsencoding, _fixencoding" "fname = '/path/to/file'" "for i in range(10000):" "    if _fixencoding: fname = _watchmantofsencoding(fname)"
Windows (_fixencoding is True): 100 loops, best of 3: 19.5 msec per loop
Linux (_fixencoding is False): 100 loops, best of 3: 3.08 msec per loop

31845:86246530b8d2

# The issues related to nested repos and subrepos are probably not fundamental
# ones. Patches to fix them are welcome.

from __future__ import absolute_import

import hashlib
import os
import stat

from mercurial.i18n import _
from mercurial import (
    context,
    encoding,
    extensions,
    localrepo,
    merge,
    pathutil,
    pycompat,

    util,
)
from mercurial import match as matchmod

from . import (
    state,
    watchmanclient,
)

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

    if util.safehasattr(ignore, '_files'):
        for f in ignore._files:
            sha1.update(f)
    sha1.update('\0')
    return sha1.hexdigest()

def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
    '''Replacement for dirstate.walk, hooking into Watchman.

    Whenever full is False, ignored is False, and the Watchman client is

    # as being happens-after the exists=False entries due to the way that
    # Watchman tracks files.  We use this property to reconcile deletes
    # for name case changes.
    for entry in result['files']:
        fname = entry['name']
        if switch_slashes:
            fname = fname.replace('\\', '/')
        if normalize:
            normed = normcase(fname)
            fname = normalize(fname, True, True)

31846:1064a296a2a7

# The issues related to nested repos and subrepos are probably not fundamental
# ones. Patches to fix them are welcome.

from __future__ import absolute_import

import codecs
import hashlib
import os
import stat
import sys

from mercurial.i18n import _
from mercurial import (
    context,
    encoding,
    error,
    extensions,
    localrepo,
    merge,
    pathutil,
    pycompat,

    util,
)
from mercurial import match as matchmod

from . import (
    pywatchman,
    state,
    watchmanclient,
)

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

    if util.safehasattr(ignore, '_files'):
        for f in ignore._files:
            sha1.update(f)
    sha1.update('\0')
    return sha1.hexdigest()

_watchmanencoding = pywatchman.encoding.get_local_encoding()
_fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
_fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)

def _watchmantofsencoding(path):
    """Fix path to match watchman and local filesystem encoding

    watchman's paths encoding can differ from filesystem encoding. For example,
    on Windows, it's always utf-8.
    """
    try:
        decoded = path.decode(_watchmanencoding)
    except UnicodeDecodeError as e:
        raise error.Abort(str(e), hint='watchman encoding error')

    try:
        encoded = decoded.encode(_fsencoding, 'strict')
    except UnicodeEncodeError as e:
        raise error.Abort(str(e))

    return encoded

def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
    '''Replacement for dirstate.walk, hooking into Watchman.

    Whenever full is False, ignored is False, and the Watchman client is

    # as being happens-after the exists=False entries due to the way that
    # Watchman tracks files.  We use this property to reconcile deletes
    # for name case changes.
    for entry in result['files']:
        fname = entry['name']
        if _fixencoding:
            fname = _watchmantofsencoding(fname)
        if switch_slashes:
            fname = fname.replace('\\', '/')
        if normalize:
            normed = normcase(fname)
            fname = normalize(fname, True, True)