mercurial/encoding.py

26878:d7e83f106459

    arbitrary bytes into an internal Unicode format that can be
    re-encoded back into the original. Here we are exposing the
    internal surrogate encoding as a UTF-8 string.)
    '''

    if isinstance(s, localstr):
        return s._utf8

    try:
        s.decode('utf-8')
        return s
    except UnicodeDecodeError:
        pass

    r = ""
    pos = 0
    l = len(s)
    while pos < l:
        try:
            c = getutf8char(s, pos)
            pos += len(c)
        except UnicodeDecodeError:
            c = unichr(0xdc00 + ord(s[pos])).encode('utf-8')
            pos += 1
        r += c
    return r

26879:a24b98f4e03c

    arbitrary bytes into an internal Unicode format that can be
    re-encoded back into the original. Here we are exposing the
    internal surrogate encoding as a UTF-8 string.)
    '''

    if "\xed" not in s:
        if isinstance(s, localstr):
            return s._utf8
        try:
            s.decode('utf-8')
            return s
        except UnicodeDecodeError:
            pass

    r = ""
    pos = 0
    l = len(s)
    while pos < l:
        try:
            c = getutf8char(s, pos)
            if "\xed\xb0\x80" <= c <= "\xed\xb3\xbf":
                # have to re-escape existing U+DCxx characters
                c = unichr(0xdc00 + ord(s[pos])).encode('utf-8')
                pos += 1
            else:
                pos += len(c)
        except UnicodeDecodeError:
            c = unichr(0xdc00 + ord(s[pos])).encode('utf-8')
            pos += 1
        r += c
    return r