mercurial/util.py

43076:2372284d9457

# python 2.6 still have deprecation warning enabled by default. We do not want
# to display anything to standard user so detect if we are running test and
# only use python deprecation warning in this case.
_dowarn = bool(encoding.environ.get('HGEMITWARNINGS'))
if _dowarn:
    # explicitly unfilter our warning for python 2.7
    #
    # The option of setting PYTHONWARNINGS in the test runner was investigated.
    # However, module name set through PYTHONWARNINGS was exactly matched, so

    This is a noop outside of tests, use 'ui.deprecwarn' when possible.
    """
    if _dowarn:
        msg += (
            "\n(compatibility will be dropped after Mercurial-%s,"
            " update your code.)"
        ) % version
        warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)


DIGESTS = {
    'md5': hashlib.md5,
    'sha1': hashlib.sha1,
    'sha512': hashlib.sha512,
}
# List of digest types from strongest to weakest
DIGESTS_BY_STRENGTH = ['sha512', 'sha1', 'md5']

for k in DIGESTS_BY_STRENGTH:
    assert k in DIGESTS

    '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
    >>> digester.preferred([b'md5', b'sha1'])
    'sha1'
    """

    def __init__(self, digests, s=''):
        self._hashes = {}
        for k in digests:
            if k not in DIGESTS:
                raise error.Abort(_('unknown digest type: %s') % k)
            self._hashes[k] = DIGESTS[k]()
        if s:
            self.update(s)

    def update(self, data):
        for h in self._hashes.values():
            h.update(data)

    def __getitem__(self, key):
        if key not in DIGESTS:
            raise error.Abort(_('unknown digest type: %s') % k)
        return nodemod.hex(self._hashes[key].digest())

    def __iter__(self):
        return iter(self._hashes)

        return content

    def validate(self):
        if self._size != self._got:
            raise error.Abort(
                _('size mismatch: expected %d, got %d')
                % (self._size, self._got)
            )
        for k, v in self._digests.items():
            if v != self._digester[k]:
                # i18n: first parameter is a digest name
                raise error.Abort(
                    _('%s mismatch: expected %s, got %s')
                    % (k, v, self._digester[k])
                )


try:

    def readline(self, *args, **kwargs):
        if len(self._buffer) > 1:
            # this should not happen because both read and readline end with a
            # _frombuffer call that collapse it.
            self._buffer = [''.join(self._buffer)]
            self._lenbuf = len(self._buffer[0])
        lfi = -1
        if self._buffer:
            lfi = self._buffer[-1].find('\n')
        while (not self._eof) and lfi < 0:
            self._fillbuffer()
            if self._buffer:
                lfi = self._buffer[-1].find('\n')
        size = lfi + 1
        if lfi < 0:  # end of file
            size = self._lenbuf
        elif len(self._buffer) > 1:
            # we need to take previous chunks into account

    def _frombuffer(self, size):
        """return at most 'size' data from the buffer

        The data are removed from the buffer."""
        if size == 0 or not self._buffer:
            return ''
        buf = self._buffer[0]
        if len(self._buffer) > 1:
            buf = ''.join(self._buffer)

        data = buf[:size]
        buf = buf[len(data) :]
        if buf:
            self._buffer = [buf]

        return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
    except ValueError:
        # Empty files cannot be mmapped, but mmapread should still work.  Check
        # if the file is empty, and if so, return an empty buffer.
        if os.fstat(fd).st_size == 0:
            return ''
        raise


class fileobjectproxy(object):
    """A proxy around file objects that tells a watcher when events occur.

class baseproxyobserver(object):
    def _writedata(self, data):
        if not self.logdata:
            if self.logdataapis:
                self.fh.write('\n')
                self.fh.flush()
            return

        # Simple case writes all data on a single line.
        if b'\n' not in data:
            if self.logdataapis:
                self.fh.write(': %s\n' % stringutil.escapestr(data))
            else:
                self.fh.write(
                    '%s>     %s\n' % (self.name, stringutil.escapestr(data))
                )
            self.fh.flush()
            return

        # Data with newlines is written to multiple lines.
        if self.logdataapis:
            self.fh.write(':\n')

        lines = data.splitlines(True)
        for line in lines:
            self.fh.write(
                '%s>     %s\n' % (self.name, stringutil.escapestr(line))
            )
        self.fh.flush()


class fileobjectobserver(baseproxyobserver):

    def read(self, res, size=-1):
        if not self.reads:
            return
        # Python 3 can return None from reads at EOF instead of empty strings.
        if res is None:
            res = ''

        if size == -1 and res == '':
            # Suppress pointless read(-1) calls that return
            # nothing. These happen _a lot_ on Python 3, and there
            # doesn't seem to be a better workaround to have matching
            # Python 2 and 3 behavior. :(
            return

        if self.logdataapis:
            self.fh.write('%s> read(%d) -> %d' % (self.name, size, len(res)))

        self._writedata(res)

    def readline(self, res, limit=-1):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write('%s> readline() -> %d' % (self.name, len(res)))

        self._writedata(res)

    def readinto(self, res, dest):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> readinto(%d) -> %r' % (self.name, len(dest), res)
            )

        data = dest[0:res] if res is not None else b''

        # _writedata() uses "in" operator and is confused by memoryview because

        # returns the integer bytes written.
        if res is None and data:
            res = len(data)

        if self.logdataapis:
            self.fh.write('%s> write(%d) -> %r' % (self.name, len(data), res))

        self._writedata(data)

    def flush(self, res):
        if not self.writes:
            return

        self.fh.write('%s> flush() -> %r\n' % (self.name, res))

    # For observedbufferedinputpipe.
    def bufferedread(self, res, size):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
            )

        self._writedata(res)

    def bufferedreadline(self, res):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> bufferedreadline() -> %d' % (self.name, len(res))
            )

        self._writedata(res)

    def makefile(self, res, mode=None, bufsize=None):
        if not self.states:
            return

        self.fh.write('%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))

    def recv(self, res, size, flags=0):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
            )
        self._writedata(res)

    def recvfrom(self, res, size, flags=0):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> recvfrom(%d, %d) -> %d'
                % (self.name, size, flags, len(res[0]))
            )

        self._writedata(res[0])

        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> recvfrom_into(%d, %d) -> %d'
                % (self.name, size, flags, res[0])
            )

        self._writedata(buf[0 : res[0]])

        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                '%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
            )

        self._writedata(buf[0:res])

    def send(self, res, data, flags=0):
        if not self.writes:
            return

        self.fh.write(
            '%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
        )
        self._writedata(data)

    def sendall(self, res, data, flags=0):
        if not self.writes:
            return

        if self.logdataapis:
            # Returns None on success. So don't bother reporting return value.
            self.fh.write('%s> sendall(%d, %d)' % (self.name, len(data), flags))

        self._writedata(data)

    def sendto(self, res, data, flagsoraddress, address=None):
        if not self.writes:

        else:
            flags = 0

        if self.logdataapis:
            self.fh.write(
                '%s> sendto(%d, %d, %r) -> %d'
                % (self.name, len(data), flags, address, res)
            )

        self._writedata(data)

    def setblocking(self, res, flag):
        if not self.states:
            return

        self.fh.write('%s> setblocking(%r)\n' % (self.name, flag))

    def settimeout(self, res, value):
        if not self.states:
            return

        self.fh.write('%s> settimeout(%r)\n' % (self.name, value))

    def gettimeout(self, res):
        if not self.states:
            return

        self.fh.write('%s> gettimeout() -> %f\n' % (self.name, res))

    def setsockopt(self, res, level, optname, value):
        if not self.states:
            return

        self.fh.write(
            '%s> setsockopt(%r, %r, %r) -> %r\n'
            % (self.name, level, optname, value, res)
        )


def makeloggingsocket(

    try:
        from . import __version__

        return __version__.version
    except ImportError:
        return 'unknown'


def versiontuple(v=None, n=4):
    """Parses a Mercurial version string into an N-tuple.

    """
    if not v:
        v = version()
    m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v)
    if not m:
        vparts, extra = '', v
    elif m.group(2):
        vparts, extra = m.groups()
    else:
        vparts, extra = m.group(1), None

    vints = []
    for i in vparts.split('.'):
        try:
            vints.append(int(i))
        except ValueError:
            break
    # (3, 6) -> (3, 6, None)

                nmin = 1 << log2(blen)
                if nmin > min:
                    min = nmin
                if min > max:
                    min = max
            yield ''.join(buf)
            blen = 0
            buf = []
    if buf:
        yield ''.join(buf)


def always(fn):
    return True

    if not n1:
        return localpath(n2)
    if os.path.isabs(n1):
        if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
            return os.path.join(root, localpath(n2))
        n2 = '/'.join((pconvert(root), n2))
    a, b = splitpath(n1), n2.split('/')
    a.reverse()
    b.reverse()
    while a and b and a[-1] == b[-1]:
        a.pop()
        b.pop()
    b.reverse()
    return pycompat.ossep.join((['..'] * len(a)) + b) or '.'


# the location of data files matching the source code
if procutil.mainfrozen() and getattr(sys, 'frozen', None) != 'macosx_app':
    # executable version (py2exe) doesn't support __file__
    datapath = os.path.dirname(pycompat.sysexecutable)
else:
    datapath = os.path.dirname(pycompat.fsencode(__file__))

    return check


# a whilelist of known filesystems where hardlink works reliably
_hardlinkfswhitelist = {
    'apfs',
    'btrfs',
    'ext2',
    'ext3',
    'ext4',
    'hfs',
    'jfs',
    'NTFS',
    'reiserfs',
    'tmpfs',
    'ufs',
    'xfs',
    'zfs',
}


def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
    '''copy a file, preserving mode and optionally other stat info like

    """Copy a directory tree using hardlinks if possible."""
    num = 0

    def settopic():
        if progress:
            progress.topic = _('linking') if hardlink else _('copying')

    if os.path.isdir(src):
        if hardlink is None:
            hardlink = (
                os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev

    return hardlink, num


_winreservednames = {
    'con',
    'prn',
    'aux',
    'nul',
    'com1',
    'com2',
    'com3',
    'com4',
    'com5',
    'com6',
    'com7',
    'com8',
    'com9',
    'lpt1',
    'lpt2',
    'lpt3',
    'lpt4',
    'lpt5',
    'lpt6',
    'lpt7',
    'lpt8',
    'lpt9',
}
_winreservedchars = ':*?"<>|'


def checkwinfilename(path):
    r'''Check that the base-relative path is a valid filename on Windows.
    Returns None if the path is ok, or a UI string describing the problem.

    >>> checkwinfilename(b"foo\\")
    "filename ends with '\\', which is invalid on Windows"
    >>> checkwinfilename(b"foo\\/bar")
    "directory name ends with '\\', which is invalid on Windows"
    '''
    if path.endswith('\\'):
        return _("filename ends with '\\', which is invalid on Windows")
    if '\\/' in path:
        return _("directory name ends with '\\', which is invalid on Windows")
    for n in path.replace('\\', '/').split('/'):
        if not n:
            continue
        for c in _filenamebytestr(n):
            if c in _winreservedchars:
                return (
                    _("filename contains '%s', which is reserved " "on Windows")
                    % c
                )
            if ord(c) <= 31:
                return _(
                    "filename contains '%s', which is invalid " "on Windows"
                ) % stringutil.escapestr(c)
        base = n.split('.')[0]
        if base and base.lower() in _winreservednames:
            return (
                _("filename contains '%s', which is reserved " "on Windows")
                % base
            )
        t = n[-1:]
        if t in '. ' and n not in '..':
            return (
                _("filename ends with '%s', which is not allowed " "on Windows")
                % t
            )


if pycompat.iswindows:

    except OSError as why:
        if why.errno not in (errno.EINVAL, errno.ENOSYS):
            raise
    except AttributeError:  # no symlink in os
        pass
    with posixfile(pathname, 'rb') as fp:
        return fp.read()


def fstat(fp):
    '''stat file object that may not have fileno method.'''

class _re(object):
    def _checkre2(self):
        global _re2
        try:
            # check if match works, see issue3964
            _re2 = bool(re2.match(r'\[([^\[]+)\]', '[ui]'))
        except ImportError:
            _re2 = False

    def compile(self, pat, flags=0):
        '''Compile a regular expression, using re2 if possible

        IGNORECASE and MULTILINE.'''
        if _re2 is None:
            self._checkre2()
        if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
            if flags & remod.IGNORECASE:
                pat = '(?i)' + pat
            if flags & remod.MULTILINE:
                pat = '(?m)' + pat
            try:
                return re2.compile(pat)
            except re2.error:
                pass
        return remod.compile(pat, flags)

    seps = pycompat.ossep
    if pycompat.osaltsep:
        seps = seps + pycompat.osaltsep
    # Protect backslashes. This gets silly very quickly.
    seps.replace('\\', '\\\\')
    pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
    dir = os.path.normpath(root)
    result = []
    for part, sep in pattern.findall(name):
        if sep:

            found = contents.get(part)

        result.append(found or part)
        dir = os.path.join(dir, part)

    return ''.join(result)


def checknlink(testfile):
    '''check whether hardlink count reporting works properly'''

    # testfile may be open, so we need a separate file for checking to
    # work around issue2543 (or testfile may get lost on Samba shares)
    f1, f2, fp = None, None, None
    try:
        fd, f1 = pycompat.mkstemp(
            prefix='.%s-' % os.path.basename(testfile),
            suffix='1~',
            dir=os.path.dirname(testfile),
        )
        os.close(fd)
        f2 = '%s2~' % f1[:-2]

        oslink(f1, f2)
        # nlinks() may behave differently for files on Windows shares if
        # the file is open.
        fp = posixfile(f2)

    can use emptyok=True as an optimization.

    Returns the name of the temporary file.
    """
    d, fn = os.path.split(name)
    fd, temp = pycompat.mkstemp(prefix='.%s-' % fn, suffix='~', dir=d)
    os.close(fd)
    # Temporary files are created with mode 0600, which is usually not
    # what we want.  If the original file already exists, just copy
    # its mode.  Otherwise, manually obey umask.
    copymode(name, temp, createmode, enforcewritable)

    if emptyok:
        return temp
    try:
        try:
            ifp = posixfile(name, "rb")
        except IOError as inst:
            if inst.errno == errno.ENOENT:
                return temp
            if not getattr(inst, 'filename', None):
                inst.filename = name
            raise
        ofp = posixfile(temp, "wb")
        for chunk in filechunkiter(ifp):
            ofp.write(chunk)
        ifp.close()
        ofp.close()
    except:  # re-raises

    checkambig argument of constructor is used with filestat, and is
    useful only if target file is guarded by any lock (e.g. repo.lock
    or repo.wlock).
    '''

    def __init__(self, name, mode='w+b', createmode=None, checkambig=False):
        self.__name = name  # permanent name
        self._tempname = mktempcopy(
            name,
            emptyok=('w' in mode),
            createmode=createmode,
            enforcewritable=('w' in mode),
        )

        self._fp = posixfile(self._tempname, mode)
        self._checkambig = checkambig

    if mode is not None:
        os.chmod(name, mode)


def readfile(path):
    with open(path, 'rb') as fp:
        return fp.read()


def writefile(path, text):
    with open(path, 'wb') as fp:
        fp.write(text)


def appendfile(path, text):
    with open(path, 'ab') as fp:
        fp.write(text)


class chunkbuffer(object):
    """Allow arbitrary sized chunks of data to be efficiently read from an

        """Read L bytes of data from the iterator of chunks of data.
        Returns less than L bytes if the iterator runs dry.

        If size parameter is omitted, read everything"""
        if l is None:
            return ''.join(self.iter)

        left = l
        buf = []
        queue = self._queue
        while left > 0:

            else:
                buf.append(chunk[offset : offset + left])
                self._chunkoffset += left
                left -= chunkremaining

        return ''.join(buf)


def filechunkiter(f, size=131072, limit=None):
    """Create a generator that produces the data in the file size
    (default 131072) bytes at a time, up to optional limit (default is

    Traceback (most recent call last):
        ...
    ParseError: fromline must be strictly positive
    """
    if toline - fromline < 0:
        raise error.ParseError(_("line range must be positive"))
    if fromline < 1:
        raise error.ParseError(_("fromline must be strictly positive"))
    return fromline - 1, toline


bytecount = unitcountfn(
    (100, 1 << 30, _('%.0f GB')),
    (10, 1 << 30, _('%.1f GB')),
    (1, 1 << 30, _('%.2f GB')),
    (100, 1 << 20, _('%.0f MB')),
    (10, 1 << 20, _('%.1f MB')),
    (1, 1 << 20, _('%.2f MB')),
    (100, 1 << 10, _('%.0f KB')),
    (10, 1 << 10, _('%.1f KB')),
    (1, 1 << 10, _('%.2f KB')),
    (1, 1, _('%.0f bytes')),
)


class transformingwriter(object):
    """Writable file wrapper to transform data by function"""

# stray CR is an error.
_eolre = remod.compile(br'\r*\n')


def tolf(s):
    return _eolre.sub('\n', s)


def tocrlf(s):
    return _eolre.sub('\r\n', s)


def _crlfwriter(fp):
    return transformingwriter(fp, tocrlf)


if pycompat.oslinesep == '\r\n':
    tonativeeol = tocrlf
    fromnativeeol = tolf
    nativeeolwriter = _crlfwriter
else:
    tonativeeol = pycompat.identity
    fromnativeeol = pycompat.identity
    nativeeolwriter = pycompat.identity

if pyplatform.python_implementation() == 'CPython' and sys.version_info < (
    3,
    0,
):
    # There is an issue in CPython that some IO methods do not handle EINTR
    # correctly. The following table shows what CPython version (and functions)

    # files approximately as "fast" files and use the fast (unsafe) code path,
    # to minimize the performance impact.
    if sys.version_info >= (2, 7, 4):
        # fp.readline deals with EINTR correctly, use it as a workaround.
        def _safeiterfile(fp):
            return iter(fp.readline, '')

    else:
        # fp.read* are broken too, manually deal with EINTR in a stupid way.
        # note: this may block longer than necessary because of bufsize.
        def _safeiterfile(fp, bufsize=4096):
            fd = fp.fileno()
            line = ''
            while True:
                try:
                    buf = os.read(fd, bufsize)
                except OSError as ex:
                    # os.read only raises EINTR before any data is read
                    if ex.errno == errno.EINTR:
                        continue
                    else:
                        raise
                line += buf
                if '\n' in buf:
                    splitted = line.splitlines(True)
                    line = ''
                    for l in splitted:
                        if l[-1] == '\n':
                            yield l
                        else:
                            line = l
                if not buf:
                    break

    escape_prefix is an optional flag that allows using doubled prefix for
    its escaping.
    """
    fn = fn or (lambda s: s)
    patterns = '|'.join(mapping.keys())
    if escape_prefix:
        patterns += '|' + prefix
        if len(prefix) > 1:
            prefix_char = prefix[1:]
        else:
            prefix_char = prefix
        mapping[prefix_char] = prefix_char

    try:
        return socket.getservbyname(pycompat.sysstr(port))
    except socket.error:
        raise error.Abort(
            _("no port number associated with service '%s'") % port
        )


class url(object):
    r"""Reliable URL parser.

    >>> url(b'http:')
    <url scheme: 'http'>
    """

    _safechars = "!~*'()+"
    _safepchars = "/!~*'()+:\\"
    _matchscheme = remod.compile('^[a-zA-Z0-9+.\\-]+:').match

    def __init__(self, path, parsequery=True, parsefragment=True):
        # We slowly chomp away at path until we have only the path left
        self.scheme = self.user = self.passwd = self.host = None
        self.port = self.path = self.query = self.fragment = None
        self._localpath = True
        self._hostport = ''
        self._origpath = path

        if parsefragment and '#' in path:
            path, self.fragment = path.split('#', 1)

        # special case for Windows drive letters and UNC paths
        if hasdriveletter(path) or path.startswith('\\\\'):
            self.path = path
            return

        # For compatibility reasons, we can't handle bundle paths as
        # normal URLS
        if path.startswith('bundle:'):
            self.scheme = 'bundle'
            path = path[7:]
            if path.startswith('//'):
                path = path[2:]
            self.path = path
            return

        if self._matchscheme(path):
            parts = path.split(':', 1)
            if parts[0]:
                self.scheme, path = parts
                self._localpath = False

        if not path:
            path = None
            if self._localpath:
                self.path = ''
                return
        else:
            if self._localpath:
                self.path = path
                return

            if parsequery and '?' in path:
                path, self.query = path.split('?', 1)
                if not path:
                    path = None
                if not self.query:
                    self.query = None

            # // is required to specify a host/authority
            if path and path.startswith('//'):
                parts = path[2:].split('/', 1)
                if len(parts) > 1:
                    self.host, path = parts
                else:
                    self.host = parts[0]
                    path = None
                if not self.host:
                    self.host = None
                    # path of file:///d is /d
                    # path of file:///d:/ is d:/, not /d:/
                    if path and not hasdriveletter(path):
                        path = '/' + path

            if self.host and '@' in self.host:
                self.user, self.host = self.host.rsplit('@', 1)
                if ':' in self.user:
                    self.user, self.passwd = self.user.split(':', 1)
                if not self.host:
                    self.host = None

            # Don't split on colons in IPv6 addresses without ports
            if (
                self.host
                and ':' in self.host
                and not (self.host.startswith('[') and self.host.endswith(']'))
            ):
                self._hostport = self.host
                self.host, self.port = self.host.rsplit(':', 1)
                if not self.host:
                    self.host = None

            if (
                self.host
                and self.scheme == 'file'
                and self.host not in ('localhost', '127.0.0.1', '[::1]')
            ):
                raise error.Abort(_('file:// URLs can only refer to localhost'))

        self.path = path

        # leave the query string escaped
        for a in ('user', 'passwd', 'host', 'port', 'path', 'fragment'):
            v = getattr(self, a)
            if v is not None:
                setattr(self, a, urlreq.unquote(v))

    @encoding.strmethod
    def __repr__(self):
        attrs = []
        for a in (
            'scheme',
            'user',
            'passwd',
            'host',
            'port',
            'path',
            'query',
            'fragment',
        ):
            v = getattr(self, a)
            if v is not None:
                attrs.append('%s: %r' % (a, pycompat.bytestr(v)))
        return '<url %s>' % ', '.join(attrs)

    def __bytes__(self):
        r"""Join the URL's components back into a URL string.

        Examples:

        >>> print(url(br'file:///D:\data\hg'))
        file:///D:\data\hg
        """
        if self._localpath:
            s = self.path
            if self.scheme == 'bundle':
                s = 'bundle:' + s
            if self.fragment:
                s += '#' + self.fragment
            return s

        s = self.scheme + ':'
        if self.user or self.passwd or self.host:
            s += '//'
        elif self.scheme and (
            not self.path
            or self.path.startswith('/')
            or hasdriveletter(self.path)
        ):
            s += '//'
            if hasdriveletter(self.path):
                s += '/'
        if self.user:
            s += urlreq.quote(self.user, safe=self._safechars)
        if self.passwd:
            s += ':' + urlreq.quote(self.passwd, safe=self._safechars)
        if self.user or self.passwd:
            s += '@'
        if self.host:
            if not (self.host.startswith('[') and self.host.endswith(']')):
                s += urlreq.quote(self.host)
            else:
                s += self.host
        if self.port:
            s += ':' + urlreq.quote(self.port)
        if self.host:
            s += '/'
        if self.path:
            # TODO: similar to the query string, we should not unescape the
            # path when we store it, the path might contain '%2f' = '/',
            # which we should *not* escape.
            s += urlreq.quote(self.path, safe=self._safepchars)
        if self.query:
            # we store the query in escaped form.
            s += '?' + self.query
        if self.fragment is not None:
            s += '#' + urlreq.quote(self.fragment, safe=self._safepchars)
        return s

    __str__ = encoding.strmethod(__bytes__)

    def authinfo(self):

            return (s, None)
        # authinfo[1] is passed to urllib2 password manager, and its
        # URIs must not contain credentials. The host is passed in the
        # URIs list because Python < 2.4.3 uses only that to search for
        # a password.
        return (s, (None, (s, self.host), self.user, self.passwd or ''))

    def isabs(self):
        if self.scheme and self.scheme != 'file':
            return True  # remote URL
        if hasdriveletter(self.path):
            return True  # absolute for our purposes - can't be joined()
        if self.path.startswith(br'\\'):
            return True  # Windows UNC path
        if self.path.startswith('/'):
            return True  # POSIX-style
        return False

    def localpath(self):
        if self.scheme == 'file' or self.scheme == 'bundle':
            path = self.path or '/'
            # For Windows, we need to promote hosts containing drive
            # letters to paths with drive letters.
            if hasdriveletter(self._hostport):
                path = self._hostport + '/' + self.path
            elif (
                self.host is not None and self.path and not hasdriveletter(path)
            ):
                path = '/' + path
            return path
        return self._origpath

    def islocal(self):
        '''whether localpath will return something that posixfile can open'''
        return (
            not self.scheme or self.scheme == 'file' or self.scheme == 'bundle'
        )


def hasscheme(path):
    return bool(url(path).scheme)


def hasdriveletter(path):
    return path and path[1:2] == ':' and path[0:1].isalpha()


def urllocalpath(path):
    return url(path, parsequery=False, parsefragment=False).localpath()

    user.

    Raises an error.Abort when the url is unsafe.
    """
    path = urlreq.unquote(path)
    if path.startswith('ssh://-') or path.startswith('svn+ssh://-'):
        raise error.Abort(
            _('potentially unsafe url: %r') % (pycompat.bytestr(path),)
        )


def hidepassword(u):
    '''hide user credential in a url string'''
    u = url(u)
    if u.passwd:
        u.passwd = '***'
    return bytes(u)


def removeauth(u):
    '''remove all authentication information from a url string'''

    u.user = u.passwd = None
    return bytes(u)


timecount = unitcountfn(
    (1, 1e3, _('%.0f s')),
    (100, 1, _('%.1f s')),
    (10, 1, _('%.2f s')),
    (1, 1, _('%.3f s')),
    (100, 0.001, _('%.1f ms')),
    (10, 0.001, _('%.2f ms')),
    (1, 0.001, _('%.3f ms')),
    (100, 0.000001, _('%.1f us')),
    (10, 0.000001, _('%.2f us')),
    (1, 0.000001, _('%.3f us')),
    (100, 0.000000001, _('%.1f ns')),
    (10, 0.000000001, _('%.2f ns')),
    (1, 0.000000001, _('%.3f ns')),
)


@attr.s
class timedcmstats(object):

    elapsed = attr.ib(default=0)
    # the number of nested timedcm context managers.
    level = attr.ib(default=1)

    def __bytes__(self):
        return timecount(self.elapsed) if self.elapsed else '<unknown>'

    __str__ = encoding.strmethod(__bytes__)


@contextlib.contextmanager

    def wrapper(*args, **kwargs):
        with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
            result = func(*args, **kwargs)
        stderr = procutil.stderr
        stderr.write(
            '%s%s: %s\n'
            % (
                ' ' * time_stats.level * 2,
                pycompat.bytestr(func.__name__),
                time_stats,
            )
        )
        return result

    return wrapper


_sizeunits = (
    ('m', 2 ** 20),
    ('k', 2 ** 10),
    ('g', 2 ** 30),
    ('kb', 2 ** 10),
    ('mb', 2 ** 20),
    ('gb', 2 ** 30),
    ('b', 1),
)


def sizetoint(s):
    '''Convert a space specifier to a byte count.

        for k, u in _sizeunits:
            if t.endswith(k):
                return int(float(t[: -len(k)]) * u)
        return int(t)
    except ValueError:
        raise error.ParseError(_("couldn't parse size: %s") % s)


class hooks(object):
    '''A collection of hook functions that can be used to extend a
    function's behavior. Hooks are called in lexicographic order,

        for source, hook in self._hooks:
            results.append(hook(*args))
        return results


def getstackframes(skip=0, line=' %-*s in %s\n', fileline='%s:%d', depth=0):
    '''Yields lines for a nicely formatted stacktrace.
    Skips the 'skip' last entries, then return the last 'depth' entries.
    Each file+linenumber is formatted according to fileline.
    Each line is formatted according to line.
    If line is None, it yields:

            else:
                yield line % (fnmax, fnln, func)


def debugstacktrace(
    msg='stacktrace', skip=0, f=procutil.stderr, otherf=procutil.stdout, depth=0
):
    '''Writes a message to f (stderr) with a nicely formatted stacktrace.
    Skips the 'skip' entries closest to the call, then show 'depth' entries.
    By default it will flush stdout first.
    It can be used everywhere and intentionally does not require an ui object.
    Not be used in production code but very convenient while developing.
    '''
    if otherf:
        otherf.flush()
    f.write('%s at:\n' % msg.rstrip())
    for line in getstackframes(skip + 1, depth=depth):
        f.write(line)
    f.flush()

            for f, s in map.iteritems():
                if s[0] != skip:
                    addpath(f)
        elif skip is not None:
            raise error.ProgrammingError(
                "skip character is only supported " "with a dict source"
            )
        else:
            for f in map:
                addpath(f)

if rustdirs is not None:
    dirs = rustdirs


def finddirs(path):
    pos = path.rfind('/')
    while pos != -1:
        yield path[:pos]
        pos = path.rfind('/', 0, pos)
    yield ''


# convenient shortcut
dst = debugstacktrace

    in the provided context and is not in the set of other names.
    """
    if others is None:
        others = set()

    fn = '%s~%s' % (f, tag)
    if fn not in ctx and fn not in others:
        return fn
    for n in itertools.count(1):
        fn = '%s~%s~%s' % (f, tag, n)
        if fn not in ctx and fn not in others:
            return fn


def readexactly(stream, n):
    '''read n bytes from stream.read and abort if less was available'''
    s = stream.read(n)
    if len(s) < n:
        raise error.Abort(
            _("stream ended unexpectedly" " (got %d bytes, expected %d)")
            % (len(s), n)
        )
    return s

    Traceback (most recent call last):
        ...
    ProgrammingError: negative value for uvarint: -1
    """
    if value < 0:
        raise error.ProgrammingError('negative value for uvarint: %d' % value)
    bits = value & 0x7F
    value >>= 7
    bytes = []
    while value:
        bytes.append(pycompat.bytechr(0x80 | bits))
        bits = value & 0x7F
        value >>= 7
    bytes.append(pycompat.bytechr(bits))

    return ''.join(bytes)


def uvarintdecodestream(fh):
    """Decode an unsigned variable length integer from a stream.

43077:687b865b95ad

# python 2.6 still have deprecation warning enabled by default. We do not want
# to display anything to standard user so detect if we are running test and
# only use python deprecation warning in this case.
_dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS'))
if _dowarn:
    # explicitly unfilter our warning for python 2.7
    #
    # The option of setting PYTHONWARNINGS in the test runner was investigated.
    # However, module name set through PYTHONWARNINGS was exactly matched, so

    This is a noop outside of tests, use 'ui.deprecwarn' when possible.
    """
    if _dowarn:
        msg += (
            b"\n(compatibility will be dropped after Mercurial-%s,"
            b" update your code.)"
        ) % version
        warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)


DIGESTS = {
    b'md5': hashlib.md5,
    b'sha1': hashlib.sha1,
    b'sha512': hashlib.sha512,
}
# List of digest types from strongest to weakest
DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5']

for k in DIGESTS_BY_STRENGTH:
    assert k in DIGESTS

    '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
    >>> digester.preferred([b'md5', b'sha1'])
    'sha1'
    """

    def __init__(self, digests, s=b''):
        self._hashes = {}
        for k in digests:
            if k not in DIGESTS:
                raise error.Abort(_(b'unknown digest type: %s') % k)
            self._hashes[k] = DIGESTS[k]()
        if s:
            self.update(s)

    def update(self, data):
        for h in self._hashes.values():
            h.update(data)

    def __getitem__(self, key):
        if key not in DIGESTS:
            raise error.Abort(_(b'unknown digest type: %s') % k)
        return nodemod.hex(self._hashes[key].digest())

    def __iter__(self):
        return iter(self._hashes)

        return content

    def validate(self):
        if self._size != self._got:
            raise error.Abort(
                _(b'size mismatch: expected %d, got %d')
                % (self._size, self._got)
            )
        for k, v in self._digests.items():
            if v != self._digester[k]:
                # i18n: first parameter is a digest name
                raise error.Abort(
                    _(b'%s mismatch: expected %s, got %s')
                    % (k, v, self._digester[k])
                )


try:

    def readline(self, *args, **kwargs):
        if len(self._buffer) > 1:
            # this should not happen because both read and readline end with a
            # _frombuffer call that collapse it.
            self._buffer = [b''.join(self._buffer)]
            self._lenbuf = len(self._buffer[0])
        lfi = -1
        if self._buffer:
            lfi = self._buffer[-1].find(b'\n')
        while (not self._eof) and lfi < 0:
            self._fillbuffer()
            if self._buffer:
                lfi = self._buffer[-1].find(b'\n')
        size = lfi + 1
        if lfi < 0:  # end of file
            size = self._lenbuf
        elif len(self._buffer) > 1:
            # we need to take previous chunks into account

    def _frombuffer(self, size):
        """return at most 'size' data from the buffer

        The data are removed from the buffer."""
        if size == 0 or not self._buffer:
            return b''
        buf = self._buffer[0]
        if len(self._buffer) > 1:
            buf = b''.join(self._buffer)

        data = buf[:size]
        buf = buf[len(data) :]
        if buf:
            self._buffer = [buf]

        return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
    except ValueError:
        # Empty files cannot be mmapped, but mmapread should still work.  Check
        # if the file is empty, and if so, return an empty buffer.
        if os.fstat(fd).st_size == 0:
            return b''
        raise


class fileobjectproxy(object):
    """A proxy around file objects that tells a watcher when events occur.

class baseproxyobserver(object):
    def _writedata(self, data):
        if not self.logdata:
            if self.logdataapis:
                self.fh.write(b'\n')
                self.fh.flush()
            return

        # Simple case writes all data on a single line.
        if b'\n' not in data:
            if self.logdataapis:
                self.fh.write(b': %s\n' % stringutil.escapestr(data))
            else:
                self.fh.write(
                    b'%s>     %s\n' % (self.name, stringutil.escapestr(data))
                )
            self.fh.flush()
            return

        # Data with newlines is written to multiple lines.
        if self.logdataapis:
            self.fh.write(b':\n')

        lines = data.splitlines(True)
        for line in lines:
            self.fh.write(
                b'%s>     %s\n' % (self.name, stringutil.escapestr(line))
            )
        self.fh.flush()


class fileobjectobserver(baseproxyobserver):

    def read(self, res, size=-1):
        if not self.reads:
            return
        # Python 3 can return None from reads at EOF instead of empty strings.
        if res is None:
            res = b''

        if size == -1 and res == b'':
            # Suppress pointless read(-1) calls that return
            # nothing. These happen _a lot_ on Python 3, and there
            # doesn't seem to be a better workaround to have matching
            # Python 2 and 3 behavior. :(
            return

        if self.logdataapis:
            self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res)))

        self._writedata(res)

    def readline(self, res, limit=-1):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(b'%s> readline() -> %d' % (self.name, len(res)))

        self._writedata(res)

    def readinto(self, res, dest):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> readinto(%d) -> %r' % (self.name, len(dest), res)
            )

        data = dest[0:res] if res is not None else b''

        # _writedata() uses "in" operator and is confused by memoryview because

        # returns the integer bytes written.
        if res is None and data:
            res = len(data)

        if self.logdataapis:
            self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res))

        self._writedata(data)

    def flush(self, res):
        if not self.writes:
            return

        self.fh.write(b'%s> flush() -> %r\n' % (self.name, res))

    # For observedbufferedinputpipe.
    def bufferedread(self, res, size):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res))
            )

        self._writedata(res)

    def bufferedreadline(self, res):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> bufferedreadline() -> %d' % (self.name, len(res))
            )

        self._writedata(res)

    def makefile(self, res, mode=None, bufsize=None):
        if not self.states:
            return

        self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize))

    def recv(self, res, size, flags=0):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res))
            )
        self._writedata(res)

    def recvfrom(self, res, size, flags=0):
        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> recvfrom(%d, %d) -> %d'
                % (self.name, size, flags, len(res[0]))
            )

        self._writedata(res[0])

        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> recvfrom_into(%d, %d) -> %d'
                % (self.name, size, flags, res[0])
            )

        self._writedata(buf[0 : res[0]])

        if not self.reads:
            return

        if self.logdataapis:
            self.fh.write(
                b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res)
            )

        self._writedata(buf[0:res])

    def send(self, res, data, flags=0):
        if not self.writes:
            return

        self.fh.write(
            b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res))
        )
        self._writedata(data)

    def sendall(self, res, data, flags=0):
        if not self.writes:
            return

        if self.logdataapis:
            # Returns None on success. So don't bother reporting return value.
            self.fh.write(
                b'%s> sendall(%d, %d)' % (self.name, len(data), flags)
            )

        self._writedata(data)

    def sendto(self, res, data, flagsoraddress, address=None):
        if not self.writes:

        else:
            flags = 0

        if self.logdataapis:
            self.fh.write(
                b'%s> sendto(%d, %d, %r) -> %d'
                % (self.name, len(data), flags, address, res)
            )

        self._writedata(data)

    def setblocking(self, res, flag):
        if not self.states:
            return

        self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag))

    def settimeout(self, res, value):
        if not self.states:
            return

        self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value))

    def gettimeout(self, res):
        if not self.states:
            return

        self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res))

    def setsockopt(self, res, level, optname, value):
        if not self.states:
            return

        self.fh.write(
            b'%s> setsockopt(%r, %r, %r) -> %r\n'
            % (self.name, level, optname, value, res)
        )


def makeloggingsocket(

    try:
        from . import __version__

        return __version__.version
    except ImportError:
        return b'unknown'


def versiontuple(v=None, n=4):
    """Parses a Mercurial version string into an N-tuple.

    """
    if not v:
        v = version()
    m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v)
    if not m:
        vparts, extra = b'', v
    elif m.group(2):
        vparts, extra = m.groups()
    else:
        vparts, extra = m.group(1), None

    vints = []
    for i in vparts.split(b'.'):
        try:
            vints.append(int(i))
        except ValueError:
            break
    # (3, 6) -> (3, 6, None)

                nmin = 1 << log2(blen)
                if nmin > min:
                    min = nmin
                if min > max:
                    min = max
            yield b''.join(buf)
            blen = 0
            buf = []
    if buf:
        yield b''.join(buf)


def always(fn):
    return True

    if not n1:
        return localpath(n2)
    if os.path.isabs(n1):
        if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
            return os.path.join(root, localpath(n2))
        n2 = b'/'.join((pconvert(root), n2))
    a, b = splitpath(n1), n2.split(b'/')
    a.reverse()
    b.reverse()
    while a and b and a[-1] == b[-1]:
        a.pop()
        b.pop()
    b.reverse()
    return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.'


# the location of data files matching the source code
if procutil.mainfrozen() and getattr(sys, 'frozen', None) != b'macosx_app':
    # executable version (py2exe) doesn't support __file__
    datapath = os.path.dirname(pycompat.sysexecutable)
else:
    datapath = os.path.dirname(pycompat.fsencode(__file__))

    return check


# a whilelist of known filesystems where hardlink works reliably
_hardlinkfswhitelist = {
    b'apfs',
    b'btrfs',
    b'ext2',
    b'ext3',
    b'ext4',
    b'hfs',
    b'jfs',
    b'NTFS',
    b'reiserfs',
    b'tmpfs',
    b'ufs',
    b'xfs',
    b'zfs',
}


def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
    '''copy a file, preserving mode and optionally other stat info like

    """Copy a directory tree using hardlinks if possible."""
    num = 0

    def settopic():
        if progress:
            progress.topic = _(b'linking') if hardlink else _(b'copying')

    if os.path.isdir(src):
        if hardlink is None:
            hardlink = (
                os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev

    return hardlink, num


_winreservednames = {
    b'con',
    b'prn',
    b'aux',
    b'nul',
    b'com1',
    b'com2',
    b'com3',
    b'com4',
    b'com5',
    b'com6',
    b'com7',
    b'com8',
    b'com9',
    b'lpt1',
    b'lpt2',
    b'lpt3',
    b'lpt4',
    b'lpt5',
    b'lpt6',
    b'lpt7',
    b'lpt8',
    b'lpt9',
}
_winreservedchars = b':*?"<>|'


def checkwinfilename(path):
    r'''Check that the base-relative path is a valid filename on Windows.
    Returns None if the path is ok, or a UI string describing the problem.

    >>> checkwinfilename(b"foo\\")
    "filename ends with '\\', which is invalid on Windows"
    >>> checkwinfilename(b"foo\\/bar")
    "directory name ends with '\\', which is invalid on Windows"
    '''
    if path.endswith(b'\\'):
        return _(b"filename ends with '\\', which is invalid on Windows")
    if b'\\/' in path:
        return _(b"directory name ends with '\\', which is invalid on Windows")
    for n in path.replace(b'\\', b'/').split(b'/'):
        if not n:
            continue
        for c in _filenamebytestr(n):
            if c in _winreservedchars:
                return (
                    _(
                        b"filename contains '%s', which is reserved "
                        b"on Windows"
                    )
                    % c
                )
            if ord(c) <= 31:
                return _(
                    b"filename contains '%s', which is invalid " b"on Windows"
                ) % stringutil.escapestr(c)
        base = n.split(b'.')[0]
        if base and base.lower() in _winreservednames:
            return (
                _(b"filename contains '%s', which is reserved " b"on Windows")
                % base
            )
        t = n[-1:]
        if t in b'. ' and n not in b'..':
            return (
                _(
                    b"filename ends with '%s', which is not allowed "
                    b"on Windows"
                )
                % t
            )


if pycompat.iswindows:

    except OSError as why:
        if why.errno not in (errno.EINVAL, errno.ENOSYS):
            raise
    except AttributeError:  # no symlink in os
        pass
    with posixfile(pathname, b'rb') as fp:
        return fp.read()


def fstat(fp):
    '''stat file object that may not have fileno method.'''

class _re(object):
    def _checkre2(self):
        global _re2
        try:
            # check if match works, see issue3964
            _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]'))
        except ImportError:
            _re2 = False

    def compile(self, pat, flags=0):
        '''Compile a regular expression, using re2 if possible

        IGNORECASE and MULTILINE.'''
        if _re2 is None:
            self._checkre2()
        if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
            if flags & remod.IGNORECASE:
                pat = b'(?i)' + pat
            if flags & remod.MULTILINE:
                pat = b'(?m)' + pat
            try:
                return re2.compile(pat)
            except re2.error:
                pass
        return remod.compile(pat, flags)

    seps = pycompat.ossep
    if pycompat.osaltsep:
        seps = seps + pycompat.osaltsep
    # Protect backslashes. This gets silly very quickly.
    seps.replace(b'\\', b'\\\\')
    pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
    dir = os.path.normpath(root)
    result = []
    for part, sep in pattern.findall(name):
        if sep:

            found = contents.get(part)

        result.append(found or part)
        dir = os.path.join(dir, part)

    return b''.join(result)


def checknlink(testfile):
    '''check whether hardlink count reporting works properly'''

    # testfile may be open, so we need a separate file for checking to
    # work around issue2543 (or testfile may get lost on Samba shares)
    f1, f2, fp = None, None, None
    try:
        fd, f1 = pycompat.mkstemp(
            prefix=b'.%s-' % os.path.basename(testfile),
            suffix=b'1~',
            dir=os.path.dirname(testfile),
        )
        os.close(fd)
        f2 = b'%s2~' % f1[:-2]

        oslink(f1, f2)
        # nlinks() may behave differently for files on Windows shares if
        # the file is open.
        fp = posixfile(f2)

    can use emptyok=True as an optimization.

    Returns the name of the temporary file.
    """
    d, fn = os.path.split(name)
    fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d)
    os.close(fd)
    # Temporary files are created with mode 0600, which is usually not
    # what we want.  If the original file already exists, just copy
    # its mode.  Otherwise, manually obey umask.
    copymode(name, temp, createmode, enforcewritable)

    if emptyok:
        return temp
    try:
        try:
            ifp = posixfile(name, b"rb")
        except IOError as inst:
            if inst.errno == errno.ENOENT:
                return temp
            if not getattr(inst, 'filename', None):
                inst.filename = name
            raise
        ofp = posixfile(temp, b"wb")
        for chunk in filechunkiter(ifp):
            ofp.write(chunk)
        ifp.close()
        ofp.close()
    except:  # re-raises

    checkambig argument of constructor is used with filestat, and is
    useful only if target file is guarded by any lock (e.g. repo.lock
    or repo.wlock).
    '''

    def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False):
        self.__name = name  # permanent name
        self._tempname = mktempcopy(
            name,
            emptyok=(b'w' in mode),
            createmode=createmode,
            enforcewritable=(b'w' in mode),
        )

        self._fp = posixfile(self._tempname, mode)
        self._checkambig = checkambig

    if mode is not None:
        os.chmod(name, mode)


def readfile(path):
    with open(path, b'rb') as fp:
        return fp.read()


def writefile(path, text):
    with open(path, b'wb') as fp:
        fp.write(text)


def appendfile(path, text):
    with open(path, b'ab') as fp:
        fp.write(text)


class chunkbuffer(object):
    """Allow arbitrary sized chunks of data to be efficiently read from an

        """Read L bytes of data from the iterator of chunks of data.
        Returns less than L bytes if the iterator runs dry.

        If size parameter is omitted, read everything"""
        if l is None:
            return b''.join(self.iter)

        left = l
        buf = []
        queue = self._queue
        while left > 0:

            else:
                buf.append(chunk[offset : offset + left])
                self._chunkoffset += left
                left -= chunkremaining

        return b''.join(buf)


def filechunkiter(f, size=131072, limit=None):
    """Create a generator that produces the data in the file size
    (default 131072) bytes at a time, up to optional limit (default is

    Traceback (most recent call last):
        ...
    ParseError: fromline must be strictly positive
    """
    if toline - fromline < 0:
        raise error.ParseError(_(b"line range must be positive"))
    if fromline < 1:
        raise error.ParseError(_(b"fromline must be strictly positive"))
    return fromline - 1, toline


bytecount = unitcountfn(
    (100, 1 << 30, _(b'%.0f GB')),
    (10, 1 << 30, _(b'%.1f GB')),
    (1, 1 << 30, _(b'%.2f GB')),
    (100, 1 << 20, _(b'%.0f MB')),
    (10, 1 << 20, _(b'%.1f MB')),
    (1, 1 << 20, _(b'%.2f MB')),
    (100, 1 << 10, _(b'%.0f KB')),
    (10, 1 << 10, _(b'%.1f KB')),
    (1, 1 << 10, _(b'%.2f KB')),
    (1, 1, _(b'%.0f bytes')),
)


class transformingwriter(object):
    """Writable file wrapper to transform data by function"""

# stray CR is an error.
_eolre = remod.compile(br'\r*\n')


def tolf(s):
    return _eolre.sub(b'\n', s)


def tocrlf(s):
    return _eolre.sub(b'\r\n', s)


def _crlfwriter(fp):
    return transformingwriter(fp, tocrlf)


if pycompat.oslinesep == b'\r\n':
    tonativeeol = tocrlf
    fromnativeeol = tolf
    nativeeolwriter = _crlfwriter
else:
    tonativeeol = pycompat.identity
    fromnativeeol = pycompat.identity
    nativeeolwriter = pycompat.identity

if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
    3,
    0,
):
    # There is an issue in CPython that some IO methods do not handle EINTR
    # correctly. The following table shows what CPython version (and functions)

    # files approximately as "fast" files and use the fast (unsafe) code path,
    # to minimize the performance impact.
    if sys.version_info >= (2, 7, 4):
        # fp.readline deals with EINTR correctly, use it as a workaround.
        def _safeiterfile(fp):
            return iter(fp.readline, b'')

    else:
        # fp.read* are broken too, manually deal with EINTR in a stupid way.
        # note: this may block longer than necessary because of bufsize.
        def _safeiterfile(fp, bufsize=4096):
            fd = fp.fileno()
            line = b''
            while True:
                try:
                    buf = os.read(fd, bufsize)
                except OSError as ex:
                    # os.read only raises EINTR before any data is read
                    if ex.errno == errno.EINTR:
                        continue
                    else:
                        raise
                line += buf
                if b'\n' in buf:
                    splitted = line.splitlines(True)
                    line = b''
                    for l in splitted:
                        if l[-1] == b'\n':
                            yield l
                        else:
                            line = l
                if not buf:
                    break

    escape_prefix is an optional flag that allows using doubled prefix for
    its escaping.
    """
    fn = fn or (lambda s: s)
    patterns = b'|'.join(mapping.keys())
    if escape_prefix:
        patterns += b'|' + prefix
        if len(prefix) > 1:
            prefix_char = prefix[1:]
        else:
            prefix_char = prefix
        mapping[prefix_char] = prefix_char

    try:
        return socket.getservbyname(pycompat.sysstr(port))
    except socket.error:
        raise error.Abort(
            _(b"no port number associated with service '%s'") % port
        )


class url(object):
    r"""Reliable URL parser.

    >>> url(b'http:')
    <url scheme: 'http'>
    """

    _safechars = b"!~*'()+"
    _safepchars = b"/!~*'()+:\\"
    _matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match

    def __init__(self, path, parsequery=True, parsefragment=True):
        # We slowly chomp away at path until we have only the path left
        self.scheme = self.user = self.passwd = self.host = None
        self.port = self.path = self.query = self.fragment = None
        self._localpath = True
        self._hostport = b''
        self._origpath = path

        if parsefragment and b'#' in path:
            path, self.fragment = path.split(b'#', 1)

        # special case for Windows drive letters and UNC paths
        if hasdriveletter(path) or path.startswith(b'\\\\'):
            self.path = path
            return

        # For compatibility reasons, we can't handle bundle paths as
        # normal URLS
        if path.startswith(b'bundle:'):
            self.scheme = b'bundle'
            path = path[7:]
            if path.startswith(b'//'):
                path = path[2:]
            self.path = path
            return

        if self._matchscheme(path):
            parts = path.split(b':', 1)
            if parts[0]:
                self.scheme, path = parts
                self._localpath = False

        if not path:
            path = None
            if self._localpath:
                self.path = b''
                return
        else:
            if self._localpath:
                self.path = path
                return

            if parsequery and b'?' in path:
                path, self.query = path.split(b'?', 1)
                if not path:
                    path = None
                if not self.query:
                    self.query = None

            # // is required to specify a host/authority
            if path and path.startswith(b'//'):
                parts = path[2:].split(b'/', 1)
                if len(parts) > 1:
                    self.host, path = parts
                else:
                    self.host = parts[0]
                    path = None
                if not self.host:
                    self.host = None
                    # path of file:///d is /d
                    # path of file:///d:/ is d:/, not /d:/
                    if path and not hasdriveletter(path):
                        path = b'/' + path

            if self.host and b'@' in self.host:
                self.user, self.host = self.host.rsplit(b'@', 1)
                if b':' in self.user:
                    self.user, self.passwd = self.user.split(b':', 1)
                if not self.host:
                    self.host = None

            # Don't split on colons in IPv6 addresses without ports
            if (
                self.host
                and b':' in self.host
                and not (
                    self.host.startswith(b'[') and self.host.endswith(b']')
                )
            ):
                self._hostport = self.host
                self.host, self.port = self.host.rsplit(b':', 1)
                if not self.host:
                    self.host = None

            if (
                self.host
                and self.scheme == b'file'
                and self.host not in (b'localhost', b'127.0.0.1', b'[::1]')
            ):
                raise error.Abort(
                    _(b'file:// URLs can only refer to localhost')
                )

        self.path = path

        # leave the query string escaped
        for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'):
            v = getattr(self, a)
            if v is not None:
                setattr(self, a, urlreq.unquote(v))

    @encoding.strmethod
    def __repr__(self):
        attrs = []
        for a in (
            b'scheme',
            b'user',
            b'passwd',
            b'host',
            b'port',
            b'path',
            b'query',
            b'fragment',
        ):
            v = getattr(self, a)
            if v is not None:
                attrs.append(b'%s: %r' % (a, pycompat.bytestr(v)))
        return b'<url %s>' % b', '.join(attrs)

    def __bytes__(self):
        r"""Join the URL's components back into a URL string.

        Examples:

        >>> print(url(br'file:///D:\data\hg'))
        file:///D:\data\hg
        """
        if self._localpath:
            s = self.path
            if self.scheme == b'bundle':
                s = b'bundle:' + s
            if self.fragment:
                s += b'#' + self.fragment
            return s

        s = self.scheme + b':'
        if self.user or self.passwd or self.host:
            s += b'//'
        elif self.scheme and (
            not self.path
            or self.path.startswith(b'/')
            or hasdriveletter(self.path)
        ):
            s += b'//'
            if hasdriveletter(self.path):
                s += b'/'
        if self.user:
            s += urlreq.quote(self.user, safe=self._safechars)
        if self.passwd:
            s += b':' + urlreq.quote(self.passwd, safe=self._safechars)
        if self.user or self.passwd:
            s += b'@'
        if self.host:
            if not (self.host.startswith(b'[') and self.host.endswith(b']')):
                s += urlreq.quote(self.host)
            else:
                s += self.host
        if self.port:
            s += b':' + urlreq.quote(self.port)
        if self.host:
            s += b'/'
        if self.path:
            # TODO: similar to the query string, we should not unescape the
            # path when we store it, the path might contain '%2f' = '/',
            # which we should *not* escape.
            s += urlreq.quote(self.path, safe=self._safepchars)
        if self.query:
            # we store the query in escaped form.
            s += b'?' + self.query
        if self.fragment is not None:
            s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars)
        return s

    __str__ = encoding.strmethod(__bytes__)

    def authinfo(self):

            return (s, None)
        # authinfo[1] is passed to urllib2 password manager, and its
        # URIs must not contain credentials. The host is passed in the
        # URIs list because Python < 2.4.3 uses only that to search for
        # a password.
        return (s, (None, (s, self.host), self.user, self.passwd or b''))

    def isabs(self):
        if self.scheme and self.scheme != b'file':
            return True  # remote URL
        if hasdriveletter(self.path):
            return True  # absolute for our purposes - can't be joined()
        if self.path.startswith(br'\\'):
            return True  # Windows UNC path
        if self.path.startswith(b'/'):
            return True  # POSIX-style
        return False

    def localpath(self):
        if self.scheme == b'file' or self.scheme == b'bundle':
            path = self.path or b'/'
            # For Windows, we need to promote hosts containing drive
            # letters to paths with drive letters.
            if hasdriveletter(self._hostport):
                path = self._hostport + b'/' + self.path
            elif (
                self.host is not None and self.path and not hasdriveletter(path)
            ):
                path = b'/' + path
            return path
        return self._origpath

    def islocal(self):
        '''whether localpath will return something that posixfile can open'''
        return (
            not self.scheme
            or self.scheme == b'file'
            or self.scheme == b'bundle'
        )


def hasscheme(path):
    return bool(url(path).scheme)


def hasdriveletter(path):
    return path and path[1:2] == b':' and path[0:1].isalpha()


def urllocalpath(path):
    return url(path, parsequery=False, parsefragment=False).localpath()

    user.

    Raises an error.Abort when the url is unsafe.
    """
    path = urlreq.unquote(path)
    if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'):
        raise error.Abort(
            _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),)
        )


def hidepassword(u):
    '''hide user credential in a url string'''
    u = url(u)
    if u.passwd:
        u.passwd = b'***'
    return bytes(u)


def removeauth(u):
    '''remove all authentication information from a url string'''

    u.user = u.passwd = None
    return bytes(u)


timecount = unitcountfn(
    (1, 1e3, _(b'%.0f s')),
    (100, 1, _(b'%.1f s')),
    (10, 1, _(b'%.2f s')),
    (1, 1, _(b'%.3f s')),
    (100, 0.001, _(b'%.1f ms')),
    (10, 0.001, _(b'%.2f ms')),
    (1, 0.001, _(b'%.3f ms')),
    (100, 0.000001, _(b'%.1f us')),
    (10, 0.000001, _(b'%.2f us')),
    (1, 0.000001, _(b'%.3f us')),
    (100, 0.000000001, _(b'%.1f ns')),
    (10, 0.000000001, _(b'%.2f ns')),
    (1, 0.000000001, _(b'%.3f ns')),
)


@attr.s
class timedcmstats(object):

    elapsed = attr.ib(default=0)
    # the number of nested timedcm context managers.
    level = attr.ib(default=1)

    def __bytes__(self):
        return timecount(self.elapsed) if self.elapsed else b'<unknown>'

    __str__ = encoding.strmethod(__bytes__)


@contextlib.contextmanager

    def wrapper(*args, **kwargs):
        with timedcm(pycompat.bytestr(func.__name__)) as time_stats:
            result = func(*args, **kwargs)
        stderr = procutil.stderr
        stderr.write(
            b'%s%s: %s\n'
            % (
                b' ' * time_stats.level * 2,
                pycompat.bytestr(func.__name__),
                time_stats,
            )
        )
        return result

    return wrapper


_sizeunits = (
    (b'm', 2 ** 20),
    (b'k', 2 ** 10),
    (b'g', 2 ** 30),
    (b'kb', 2 ** 10),
    (b'mb', 2 ** 20),
    (b'gb', 2 ** 30),
    (b'b', 1),
)


def sizetoint(s):
    '''Convert a space specifier to a byte count.

        for k, u in _sizeunits:
            if t.endswith(k):
                return int(float(t[: -len(k)]) * u)
        return int(t)
    except ValueError:
        raise error.ParseError(_(b"couldn't parse size: %s") % s)


class hooks(object):
    '''A collection of hook functions that can be used to extend a
    function's behavior. Hooks are called in lexicographic order,

        for source, hook in self._hooks:
            results.append(hook(*args))
        return results


def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0):
    '''Yields lines for a nicely formatted stacktrace.
    Skips the 'skip' last entries, then return the last 'depth' entries.
    Each file+linenumber is formatted according to fileline.
    Each line is formatted according to line.
    If line is None, it yields:

            else:
                yield line % (fnmax, fnln, func)


def debugstacktrace(
    msg=b'stacktrace',
    skip=0,
    f=procutil.stderr,
    otherf=procutil.stdout,
    depth=0,
):
    '''Writes a message to f (stderr) with a nicely formatted stacktrace.
    Skips the 'skip' entries closest to the call, then show 'depth' entries.
    By default it will flush stdout first.
    It can be used everywhere and intentionally does not require an ui object.
    Not be used in production code but very convenient while developing.
    '''
    if otherf:
        otherf.flush()
    f.write(b'%s at:\n' % msg.rstrip())
    for line in getstackframes(skip + 1, depth=depth):
        f.write(line)
    f.flush()

            for f, s in map.iteritems():
                if s[0] != skip:
                    addpath(f)
        elif skip is not None:
            raise error.ProgrammingError(
                b"skip character is only supported " b"with a dict source"
            )
        else:
            for f in map:
                addpath(f)

if rustdirs is not None:
    dirs = rustdirs


def finddirs(path):
    pos = path.rfind(b'/')
    while pos != -1:
        yield path[:pos]
        pos = path.rfind(b'/', 0, pos)
    yield b''


# convenient shortcut
dst = debugstacktrace

    in the provided context and is not in the set of other names.
    """
    if others is None:
        others = set()

    fn = b'%s~%s' % (f, tag)
    if fn not in ctx and fn not in others:
        return fn
    for n in itertools.count(1):
        fn = b'%s~%s~%s' % (f, tag, n)
        if fn not in ctx and fn not in others:
            return fn


def readexactly(stream, n):
    '''read n bytes from stream.read and abort if less was available'''
    s = stream.read(n)
    if len(s) < n:
        raise error.Abort(
            _(b"stream ended unexpectedly" b" (got %d bytes, expected %d)")
            % (len(s), n)
        )
    return s

    Traceback (most recent call last):
        ...
    ProgrammingError: negative value for uvarint: -1
    """
    if value < 0:
        raise error.ProgrammingError(b'negative value for uvarint: %d' % value)
    bits = value & 0x7F
    value >>= 7
    bytes = []
    while value:
        bytes.append(pycompat.bytechr(0x80 | bits))
        bits = value & 0x7F
        value >>= 7
    bytes.append(pycompat.bytechr(bits))

    return b''.join(bytes)


def uvarintdecodestream(fh):
    """Decode an unsigned variable length integer from a stream.