comparison mercurial/keepalive.py @ 43077:687b865b95ad

formatting: byteify all mercurial/ and hgext/ string literals

Done with

    python3.7 contrib/byteify-strings.py -i $(hg files 'set:mercurial/**.py - mercurial/thirdparty/** + hgext/**.py - hgext/fsmonitor/pywatchman/** - mercurial/__init__.py')
    black -l 80 -t py33 -S $(hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**" - hgext/fsmonitor/pywatchman/**')

# skip-blame mass-reformatting only

Differential Revision: https://phab.mercurial-scm.org/D6972

43076:2372284d9457

        return self.do_open(HTTPConnection, req)

    def do_open(self, http_class, req):
        host = urllibcompat.gethost(req)
        if not host:
            raise urlerr.urlerror('no host given')

        try:
            h = self._cm.get_ready_conn(host)
            while h:
                r = self._reuse_connection(h, req, host)

            else:
                # no (working) free connections were found.  Create a new one.
                h = http_class(host, timeout=self._timeout)
                if DEBUG:
                    DEBUG.info(
                        "creating new connection to %s (%d)", host, id(h)
                    )
                self._cm.add(host, h, False)
                self._start_transaction(h, req)
                r = h.getresponse()
        # The string form of BadStatusLine is the status line. Add some context
        # to make the error message slightly more useful.
        except httplib.BadStatusLine as err:
            raise urlerr.urlerror(
                _('bad HTTP status line: %s') % pycompat.sysbytes(err.line)
            )
        except (socket.error, httplib.HTTPException) as err:
            raise urlerr.urlerror(err)

        # If not a persistent connection, don't try to reuse it. Look

        # attribute, and in that case always close the connection.
        if getattr(r, r'will_close', True):
            self._cm.remove(h)

        if DEBUG:
            DEBUG.info("STATUS: %s, %s", r.status, r.reason)
        r._handler = self
        r._host = host
        r._url = req.get_full_url()
        r._connection = h
        r.code = r.status

            # same exception was raised, etc.  The trade-off is
            # that it's now possible this call will raise
            # a DIFFERENT exception
            if DEBUG:
                DEBUG.error(
                    "unexpected exception - closing " "connection to %s (%d)",
                    host,
                    id(h),
                )
            self._cm.remove(h)
            h.close()

            # bad header back.  This is most likely to happen if
            # the socket has been closed by the server since we
            # last used the connection.
            if DEBUG:
                DEBUG.info(
                    "failed to re-use connection to %s (%d)", host, id(h)
                )
            r = None
        else:
            if DEBUG:
                DEBUG.info("re-using connection to %s (%d)", host, id(h))

        return r

    def _start_transaction(self, h, req):
        oldbytescount = getattr(h, 'sentbytescount', 0)

            self, sock, debuglevel=debuglevel, method=method, **extrakw
        )
        self.fileno = sock.fileno
        self.code = None
        self.receivedbytescount = 0
        self._rbuf = ''
        self._rbufsize = 8096
        self._handler = None  # inserted by the handler later
        self._host = None  # (same)
        self._url = None  # (same)
        self._connection = None  # (same)

                return s
        # Careful! http.client.HTTPResponse.read() on Python 3 is
        # implemented using readinto(), which can duplicate self._rbuf
        # if it's not empty.
        s = self._rbuf
        self._rbuf = ''
        data = self._raw_read(amt)

        self.receivedbytescount += len(data)
        try:
            self._connection.receivedbytescount += len(data)

        parts = []

        while True:
            if chunk_left is None:
                line = self.fp.readline()
                i = line.find(';')
                if i >= 0:
                    line = line[:i]  # strip chunk-extensions
                try:
                    chunk_left = int(line, 16)
                except ValueError:
                    # close the connection as protocol synchronization is
                    # probably lost
                    self.close()
                    raise httplib.IncompleteRead(''.join(parts))
                if chunk_left == 0:
                    break
            if amt is None:
                parts.append(self._safe_read(chunk_left))
            elif amt < chunk_left:
                parts.append(self._safe_read(amt))
                self.chunk_left = chunk_left - amt
                return ''.join(parts)
            elif amt == chunk_left:
                parts.append(self._safe_read(amt))
                self._safe_read(2)  # toss the CRLF at the end of the chunk
                self.chunk_left = None
                return ''.join(parts)
            else:
                parts.append(self._safe_read(chunk_left))
                amt -= chunk_left

            # we read the whole chunk, get another

            line = self.fp.readline()
            if not line:
                # a vanishingly small number of sites EOF without
                # sending the trailer
                break
            if line == '\r\n':
                break

        # we read everything; close the "file"
        self.close()

        return ''.join(parts)

    def readline(self):
        # Fast path for a line is already available in read buffer.
        i = self._rbuf.find('\n')
        if i >= 0:
            i += 1
            line = self._rbuf[:i]
            self._rbuf = self._rbuf[i:]
            return line

                self._handler.parent.receivedbytescount += len(new)
            except AttributeError:
                pass

            chunks.append(new)
            i = new.find('\n')
            if i >= 0:
                break

        # We either have exhausted the stream or have a newline in chunks[-1].

        # EOF
        if i == -1:
            self._rbuf = ''
            return ''.join(chunks)

        i += 1
        self._rbuf = chunks[-1][i:]
        chunks[-1] = chunks[-1][:i]
        return ''.join(chunks)

    def readlines(self, sizehint=0):
        total = 0
        list = []
        while True:

        except AttributeError:
            pass

        dest[0:have] = self._rbuf
        got += len(self._rbuf)
        self._rbuf = ''
        return got


def safesend(self, str):
    """Send `str' to the server.

    # the socket. we want to reconnect when somebody tries to send again.
    #
    # NOTE: we DO propagate the error, though, because we cannot simply
    #       ignore the error... the caller will know if they can retry.
    if self.debuglevel > 0:
        print("send:", repr(str))
    try:
        blocksize = 8192
        read = getattr(str, 'read', None)
        if read is not None:
            if self.debuglevel > 0:
                print("sending a read()able")
            data = read(blocksize)
            while data:
                self.sock.sendall(data)
                self.sentbytescount += len(data)
                data = read(blocksize)

#########################################################################


def continuity(url):
    md5 = hashlib.md5
    format = '%25s: %s'

    # first fetch the file with the normal http handler
    opener = urlreq.buildopener()
    urlreq.installopener(opener)
    fo = urlreq.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % ('normal urllib', node.hex(m.digest())))

    # now install the keepalive handler and try again
    opener = urlreq.buildopener(HTTPHandler())
    urlreq.installopener(opener)

    fo = urlreq.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % ('keepalive read', node.hex(m.digest())))

    fo = urlreq.urlopen(url)
    foo = ''
    while True:
        f = fo.readline()
        if f:
            foo = foo + f
        else:
            break
    fo.close()
    m = md5(foo)
    print(format % ('keepalive readline', node.hex(m.digest())))


def comp(N, url):
    print('  making %i connections to:\n  %s' % (N, url))

    procutil.stdout.write('  first using the normal urllib handlers')
    # first use normal opener
    opener = urlreq.buildopener()
    urlreq.installopener(opener)
    t1 = fetch(N, url)
    print('  TIME: %.3f s' % t1)

    procutil.stdout.write('  now using the keepalive handler       ')
    # now install the keepalive handler and try again
    opener = urlreq.buildopener(HTTPHandler())
    urlreq.installopener(opener)
    t2 = fetch(N, url)
    print('  TIME: %.3f s' % t2)
    print('  improvement factor: %.2f' % (t1 / t2))


def fetch(N, url, delay=0):
    import time

    j = 0
    for i in lens[1:]:
        j = j + 1
        if not i == lens[0]:
            print("WARNING: inconsistent length on read %i: %i" % (j, i))

    return diff


def test_timeout(url):

            print(msg % args)

        info = warning = error = debug

    DEBUG = FakeLogger()
    print("  fetching the file to establish a connection")
    fo = urlreq.urlopen(url)
    data1 = fo.read()
    fo.close()

    i = 20
    print("  waiting %i seconds for the server to close the connection" % i)
    while i > 0:
        procutil.stdout.write('\r  %2i' % i)
        procutil.stdout.flush()
        time.sleep(1)
        i -= 1
    procutil.stderr.write('\r')

    print("  fetching the file a second time")
    fo = urlreq.urlopen(url)
    data2 = fo.read()
    fo.close()

    if data1 == data2:
        print('  data are identical')
    else:
        print('  ERROR: DATA DIFFER')

    DEBUG = dbbackup


def test(url, N=10):
    print("performing continuity test (making sure stuff isn't corrupted)")
    continuity(url)
    print('')
    print("performing speed comparison")
    comp(N, url)
    print('')
    print("performing dropped-connection check")
    test_timeout(url)


if __name__ == '__main__':
    import time

    try:
        N = int(sys.argv[1])
        url = sys.argv[2]
    except (IndexError, ValueError):
        print("%s <integer> <url>" % sys.argv[0])
    else:
        test(url, N)

43077:687b865b95ad

        return self.do_open(HTTPConnection, req)

    def do_open(self, http_class, req):
        host = urllibcompat.gethost(req)
        if not host:
            raise urlerr.urlerror(b'no host given')

        try:
            h = self._cm.get_ready_conn(host)
            while h:
                r = self._reuse_connection(h, req, host)

            else:
                # no (working) free connections were found.  Create a new one.
                h = http_class(host, timeout=self._timeout)
                if DEBUG:
                    DEBUG.info(
                        b"creating new connection to %s (%d)", host, id(h)
                    )
                self._cm.add(host, h, False)
                self._start_transaction(h, req)
                r = h.getresponse()
        # The string form of BadStatusLine is the status line. Add some context
        # to make the error message slightly more useful.
        except httplib.BadStatusLine as err:
            raise urlerr.urlerror(
                _(b'bad HTTP status line: %s') % pycompat.sysbytes(err.line)
            )
        except (socket.error, httplib.HTTPException) as err:
            raise urlerr.urlerror(err)

        # If not a persistent connection, don't try to reuse it. Look

        # attribute, and in that case always close the connection.
        if getattr(r, r'will_close', True):
            self._cm.remove(h)

        if DEBUG:
            DEBUG.info(b"STATUS: %s, %s", r.status, r.reason)
        r._handler = self
        r._host = host
        r._url = req.get_full_url()
        r._connection = h
        r.code = r.status

            # same exception was raised, etc.  The trade-off is
            # that it's now possible this call will raise
            # a DIFFERENT exception
            if DEBUG:
                DEBUG.error(
                    b"unexpected exception - closing " b"connection to %s (%d)",
                    host,
                    id(h),
                )
            self._cm.remove(h)
            h.close()

            # bad header back.  This is most likely to happen if
            # the socket has been closed by the server since we
            # last used the connection.
            if DEBUG:
                DEBUG.info(
                    b"failed to re-use connection to %s (%d)", host, id(h)
                )
            r = None
        else:
            if DEBUG:
                DEBUG.info(b"re-using connection to %s (%d)", host, id(h))

        return r

    def _start_transaction(self, h, req):
        oldbytescount = getattr(h, 'sentbytescount', 0)

            self, sock, debuglevel=debuglevel, method=method, **extrakw
        )
        self.fileno = sock.fileno
        self.code = None
        self.receivedbytescount = 0
        self._rbuf = b''
        self._rbufsize = 8096
        self._handler = None  # inserted by the handler later
        self._host = None  # (same)
        self._url = None  # (same)
        self._connection = None  # (same)

                return s
        # Careful! http.client.HTTPResponse.read() on Python 3 is
        # implemented using readinto(), which can duplicate self._rbuf
        # if it's not empty.
        s = self._rbuf
        self._rbuf = b''
        data = self._raw_read(amt)

        self.receivedbytescount += len(data)
        try:
            self._connection.receivedbytescount += len(data)

        parts = []

        while True:
            if chunk_left is None:
                line = self.fp.readline()
                i = line.find(b';')
                if i >= 0:
                    line = line[:i]  # strip chunk-extensions
                try:
                    chunk_left = int(line, 16)
                except ValueError:
                    # close the connection as protocol synchronization is
                    # probably lost
                    self.close()
                    raise httplib.IncompleteRead(b''.join(parts))
                if chunk_left == 0:
                    break
            if amt is None:
                parts.append(self._safe_read(chunk_left))
            elif amt < chunk_left:
                parts.append(self._safe_read(amt))
                self.chunk_left = chunk_left - amt
                return b''.join(parts)
            elif amt == chunk_left:
                parts.append(self._safe_read(amt))
                self._safe_read(2)  # toss the CRLF at the end of the chunk
                self.chunk_left = None
                return b''.join(parts)
            else:
                parts.append(self._safe_read(chunk_left))
                amt -= chunk_left

            # we read the whole chunk, get another

            line = self.fp.readline()
            if not line:
                # a vanishingly small number of sites EOF without
                # sending the trailer
                break
            if line == b'\r\n':
                break

        # we read everything; close the "file"
        self.close()

        return b''.join(parts)

    def readline(self):
        # Fast path for a line is already available in read buffer.
        i = self._rbuf.find(b'\n')
        if i >= 0:
            i += 1
            line = self._rbuf[:i]
            self._rbuf = self._rbuf[i:]
            return line

                self._handler.parent.receivedbytescount += len(new)
            except AttributeError:
                pass

            chunks.append(new)
            i = new.find(b'\n')
            if i >= 0:
                break

        # We either have exhausted the stream or have a newline in chunks[-1].

        # EOF
        if i == -1:
            self._rbuf = b''
            return b''.join(chunks)

        i += 1
        self._rbuf = chunks[-1][i:]
        chunks[-1] = chunks[-1][:i]
        return b''.join(chunks)

    def readlines(self, sizehint=0):
        total = 0
        list = []
        while True:

        except AttributeError:
            pass

        dest[0:have] = self._rbuf
        got += len(self._rbuf)
        self._rbuf = b''
        return got


def safesend(self, str):
    """Send `str' to the server.

    # the socket. we want to reconnect when somebody tries to send again.
    #
    # NOTE: we DO propagate the error, though, because we cannot simply
    #       ignore the error... the caller will know if they can retry.
    if self.debuglevel > 0:
        print(b"send:", repr(str))
    try:
        blocksize = 8192
        read = getattr(str, 'read', None)
        if read is not None:
            if self.debuglevel > 0:
                print(b"sending a read()able")
            data = read(blocksize)
            while data:
                self.sock.sendall(data)
                self.sentbytescount += len(data)
                data = read(blocksize)

#########################################################################


def continuity(url):
    md5 = hashlib.md5
    format = b'%25s: %s'

    # first fetch the file with the normal http handler
    opener = urlreq.buildopener()
    urlreq.installopener(opener)
    fo = urlreq.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % (b'normal urllib', node.hex(m.digest())))

    # now install the keepalive handler and try again
    opener = urlreq.buildopener(HTTPHandler())
    urlreq.installopener(opener)

    fo = urlreq.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % (b'keepalive read', node.hex(m.digest())))

    fo = urlreq.urlopen(url)
    foo = b''
    while True:
        f = fo.readline()
        if f:
            foo = foo + f
        else:
            break
    fo.close()
    m = md5(foo)
    print(format % (b'keepalive readline', node.hex(m.digest())))


def comp(N, url):
    print(b'  making %i connections to:\n  %s' % (N, url))

    procutil.stdout.write(b'  first using the normal urllib handlers')
    # first use normal opener
    opener = urlreq.buildopener()
    urlreq.installopener(opener)
    t1 = fetch(N, url)
    print(b'  TIME: %.3f s' % t1)

    procutil.stdout.write(b'  now using the keepalive handler       ')
    # now install the keepalive handler and try again
    opener = urlreq.buildopener(HTTPHandler())
    urlreq.installopener(opener)
    t2 = fetch(N, url)
    print(b'  TIME: %.3f s' % t2)
    print(b'  improvement factor: %.2f' % (t1 / t2))


def fetch(N, url, delay=0):
    import time

    j = 0
    for i in lens[1:]:
        j = j + 1
        if not i == lens[0]:
            print(b"WARNING: inconsistent length on read %i: %i" % (j, i))

    return diff


def test_timeout(url):

            print(msg % args)

        info = warning = error = debug

    DEBUG = FakeLogger()
    print(b"  fetching the file to establish a connection")
    fo = urlreq.urlopen(url)
    data1 = fo.read()
    fo.close()

    i = 20
    print(b"  waiting %i seconds for the server to close the connection" % i)
    while i > 0:
        procutil.stdout.write(b'\r  %2i' % i)
        procutil.stdout.flush()
        time.sleep(1)
        i -= 1
    procutil.stderr.write(b'\r')

    print(b"  fetching the file a second time")
    fo = urlreq.urlopen(url)
    data2 = fo.read()
    fo.close()

    if data1 == data2:
        print(b'  data are identical')
    else:
        print(b'  ERROR: DATA DIFFER')

    DEBUG = dbbackup


def test(url, N=10):
    print(b"performing continuity test (making sure stuff isn't corrupted)")
    continuity(url)
    print(b'')
    print(b"performing speed comparison")
    comp(N, url)
    print(b'')
    print(b"performing dropped-connection check")
    test_timeout(url)


if __name__ == '__main__':
    import time

    try:
        N = int(sys.argv[1])
        url = sys.argv[2]
    except (IndexError, ValueError):
        print(b"%s <integer> <url>" % sys.argv[0])
    else:
        test(url, N)