comparison mercurial/encoding.py @ 45942:89a2afe31e82

formating: upgrade to black 20.8b1

This required a couple of small tweaks to un-confuse black, but now it
works. Big formatting changes come from:

 * Dramatically improved collection-splitting logic upstream
 * Black having a strong (correct IMO) opinion that """ is better than '''

Differential Revision: https://phab.mercurial-scm.org/D9430

45941:346af7687c6f

encodingmode = environ.get(b"HGENCODINGMODE", b"strict")
fallbackencoding = b'ISO-8859-1'


class localstr(bytes):
    '''This class allows strings that are unmodified to be
    round-tripped to the local encoding and back'''

    def __new__(cls, u, l):
        s = bytes.__new__(cls, l)
        s._utf8 = u
        return s

    return len(d)


def getcols(s, start, c):
    # type: (bytes, int, int) -> bytes
    '''Use colwidth to find a c-column substring of s starting at byte
    index start'''
    for x in pycompat.xrange(start + c, len(s)):
        t = s[start:x]
        if colwidth(t) == c:
            return t
    raise ValueError('substring not found')

    except LookupError as k:
        raise error.Abort(k, hint=b"please check your locale settings")


class normcasespecs(object):
    '''what a platform's normcase does to ASCII strings

    This is specified per platform, and should be consistent with what normcase
    on that platform actually does.

    lower: normcase lowercases ASCII strings
    upper: normcase uppercases ASCII strings
    other: the fallback function should always be called

    This should be kept in sync with normcase_spec in util.h.'''

    lower = -1
    upper = 1
    other = 0


def jsonescape(s, paranoid=False):
    # type: (Any, Any) -> Any
    '''returns a string suitable for JSON

    JSON is problematic for us because it doesn't support non-Unicode
    bytes. To deal with this, we take the following approach:

    - localstr/safelocalstr objects are converted back to UTF-8

    'utf-8: caf\\\\u00e9'
    >>> jsonescape(b'non-BMP: \\xf0\\x9d\\x84\\x9e', paranoid=True)
    'non-BMP: \\\\ud834\\\\udd1e'
    >>> jsonescape(b'<[e-mail]>', paranoid=True)
    '\\\\[e-mail]\\\\u003e'
    '''

    u8chars = toutf8b(s)
    try:
        return _jsonescapeu8fast(u8chars, paranoid)
    except ValueError:

_utf8len = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4]


def getutf8char(s, pos):
    # type: (bytes, int) -> bytes
    '''get the next full utf-8 character in the given string, starting at pos

    Raises a UnicodeError if the given location does not start a valid
    utf-8 character.
    '''

    # find how many bytes to attempt decoding from first nibble
    l = _utf8len[ord(s[pos : pos + 1]) >> 4]
    if not l:  # ascii
        return s[pos : pos + 1]

    return c


def toutf8b(s):
    # type: (bytes) -> bytes
    '''convert a local, possibly-binary string into UTF-8b

    This is intended as a generic method to preserve data when working
    with schemes like JSON and XML that have no provision for
    arbitrary byte strings. As Mercurial often doesn't know
    what encoding data is in, we use so-called UTF-8b.

    (Note: "UTF-8b" often refers to decoding a mix of valid UTF-8 and
    arbitrary bytes into an internal Unicode format that can be
    re-encoded back into the original. Here we are exposing the
    internal surrogate encoding as a UTF-8 string.)
    '''

    if isinstance(s, localstr):
        # assume that the original UTF-8 sequence would never contain
        # invalid characters in U+DCxx range
        return s._utf8

    return r


def fromutf8b(s):
    # type: (bytes) -> bytes
    '''Given a UTF-8b string, return a local, possibly-binary string.

    return the original binary string. This
    is a round-trip process for strings like filenames, but metadata
    that's was passed through tolocal will remain in UTF-8.

    True
    >>> roundtrip(b"\\xef\\xef\\xbf\\xbd")
    True
    >>> roundtrip(b"\\xf1\\x80\\x80\\x80\\x80")
    True
    '''

    if isasciistr(s):
        return s
    # fast path - look for uDxxx prefixes in s
    if b"\xed" not in s:

45942:89a2afe31e82

encodingmode = environ.get(b"HGENCODINGMODE", b"strict")
fallbackencoding = b'ISO-8859-1'


class localstr(bytes):
    """This class allows strings that are unmodified to be
    round-tripped to the local encoding and back"""

    def __new__(cls, u, l):
        s = bytes.__new__(cls, l)
        s._utf8 = u
        return s

    return len(d)


def getcols(s, start, c):
    # type: (bytes, int, int) -> bytes
    """Use colwidth to find a c-column substring of s starting at byte
    index start"""
    for x in pycompat.xrange(start + c, len(s)):
        t = s[start:x]
        if colwidth(t) == c:
            return t
    raise ValueError('substring not found')

    except LookupError as k:
        raise error.Abort(k, hint=b"please check your locale settings")


class normcasespecs(object):
    """what a platform's normcase does to ASCII strings

    This is specified per platform, and should be consistent with what normcase
    on that platform actually does.

    lower: normcase lowercases ASCII strings
    upper: normcase uppercases ASCII strings
    other: the fallback function should always be called

    This should be kept in sync with normcase_spec in util.h."""

    lower = -1
    upper = 1
    other = 0


def jsonescape(s, paranoid=False):
    # type: (Any, Any) -> Any
    """returns a string suitable for JSON

    JSON is problematic for us because it doesn't support non-Unicode
    bytes. To deal with this, we take the following approach:

    - localstr/safelocalstr objects are converted back to UTF-8

    'utf-8: caf\\\\u00e9'
    >>> jsonescape(b'non-BMP: \\xf0\\x9d\\x84\\x9e', paranoid=True)
    'non-BMP: \\\\ud834\\\\udd1e'
    >>> jsonescape(b'<[e-mail]>', paranoid=True)
    '\\\\[e-mail]\\\\u003e'
    """

    u8chars = toutf8b(s)
    try:
        return _jsonescapeu8fast(u8chars, paranoid)
    except ValueError:

_utf8len = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4]


def getutf8char(s, pos):
    # type: (bytes, int) -> bytes
    """get the next full utf-8 character in the given string, starting at pos

    Raises a UnicodeError if the given location does not start a valid
    utf-8 character.
    """

    # find how many bytes to attempt decoding from first nibble
    l = _utf8len[ord(s[pos : pos + 1]) >> 4]
    if not l:  # ascii
        return s[pos : pos + 1]

    return c


def toutf8b(s):
    # type: (bytes) -> bytes
    """convert a local, possibly-binary string into UTF-8b

    This is intended as a generic method to preserve data when working
    with schemes like JSON and XML that have no provision for
    arbitrary byte strings. As Mercurial often doesn't know
    what encoding data is in, we use so-called UTF-8b.

    (Note: "UTF-8b" often refers to decoding a mix of valid UTF-8 and
    arbitrary bytes into an internal Unicode format that can be
    re-encoded back into the original. Here we are exposing the
    internal surrogate encoding as a UTF-8 string.)
    """

    if isinstance(s, localstr):
        # assume that the original UTF-8 sequence would never contain
        # invalid characters in U+DCxx range
        return s._utf8

    return r


def fromutf8b(s):
    # type: (bytes) -> bytes
    """Given a UTF-8b string, return a local, possibly-binary string.

    return the original binary string. This
    is a round-trip process for strings like filenames, but metadata
    that's was passed through tolocal will remain in UTF-8.

    True
    >>> roundtrip(b"\\xef\\xef\\xbf\\xbd")
    True
    >>> roundtrip(b"\\xf1\\x80\\x80\\x80\\x80")
    True
    """

    if isasciistr(s):
        return s
    # fast path - look for uDxxx prefixes in s
    if b"\xed" not in s: