comparison mercurial/httpclient/__init__.py @ 19182:fae47ecaa952

httpclient: upgrade to fe8c09e4db64 of httpplus

19181:8c2fdf7d5645

  * uses select() to block for incoming data
  * notices when the server responds early to a request
  * implements ssl inline instead of in a different class
"""

import cStringIO
import errno
import httplib
import logging
import rfc822

        if self._reader:
            return self._reader.done()

    def _close(self):
        if self._reader is not None:
            self._reader._close()

    def readline(self):
        """Read a single line from the response body.

            self._select()

        return ''.join(blocks)

    def read(self, length=None):
        # if length is None, unbounded read
        while (not self.complete()  # never select on a finished read
               and (not length  # unbounded, so we wait for complete()
                    or length > self._reader.available_data)):
            self._select()

        if self.complete() and self.will_close:
            self.sock.close()
        return r

    def _select(self):
        r, _, _ = select.select([self.sock], [], [], self._timeout)
        if not r:
            # socket was not readable. If the response is not
            # complete, raise a timeout.
            if not self.complete():
                logger.info('timed out with timeout of %s', self._timeout)

        # If the socket was readable and no data was read, that means
        # the socket was closed. Inform the reader (if any) so it can
        # raise an exception if this is an invalid situation.
        if not data:
            if self._reader:
                self._reader._close()
            return False
        else:
            self._load_response(data)
            return True

    def _load_response(self, data):
        # Being here implies we're not at the end of the headers yet,
        # since at the end of this method if headers were completely
        # loaded we replace this method with the load() method of the
        # reader we created.
        self.raw_response += data

        if self._end_headers not in self.raw_response or self.headers:
            return

        # handle 100-continue response
        hdrs, body = self.raw_response.split(self._end_headers, 1)
        http_ver, status = hdrs.split(' ', 1)
        if status.startswith('100'):
            self.raw_response = body
            self.continued = True
            logger.debug('continue seen, setting body to %r', body)
            return

                self._reader = _readers.CloseIsEndReader()
                logger.debug('using a close-is-end reader')
                self.will_close = True

        if body:
            self._reader._load(body)
        logger.debug('headers complete')
        self.headers = headers
        self._load_response = self._reader._load


class HTTPConnection(object):
    """Connection to a single http server.

                        self._proxy_host, self._proxy_port)
            sock = socketutil.create_connection((self._proxy_host,
                                                 self._proxy_port))
            if self.ssl:
                # TODO proxy header support
                data = self.buildheaders('CONNECT', '%s:%d' % (self.host,
                                                               self.port),
                                         {}, HTTP_VER_1_0)
                sock.send(data)
                sock.setblocking(0)
                r = self.response_class(sock, self.timeout, 'CONNECT')
                timeout_exc = HTTPTimeoutException(
                    'Timed out waiting for CONNECT response from proxy')
                while not r.complete():
                    try:
                        if not r._select():
                            if not r.complete():
                                raise timeout_exc
                    except HTTPTimeoutException:
                        # This raise/except pattern looks goofy, but

            if self._ssl_validator:
                self._ssl_validator(sock)
        sock.setblocking(0)
        self.sock = sock

    def buildheaders(self, method, path, headers, http_ver):
        if self.ssl and self.port == 443 or self.port == 80:
            # default port for protocol, so leave it out
            hdrhost = self.host
        else:
            # include nonstandard port in header

                    self._current_response = None
                    return False
            return True
        return False

    def request(self, method, path, body=None, headers={},
                expect_continue=False):
        """Send a request to the server.

        For increased flexibility, this does not return the response

                chunked = True
            else:
                raise BadRequestData('body has no __len__() nor read()')

        self._connect()
        outgoing_headers = self.buildheaders(
            method, path, hdrs, self.http_version)
        response = None
        first = True

        def reconnect(where):
            logger.info('reconnecting during %s', where)
            self.close()
            self._connect()

        while ((outgoing_headers or body)
               and not (response and response.complete())):
            select_timeout = self.timeout
            out = outgoing_headers or body

                    try:
                        data = r[0].recv(INCOMING_BUFFER_SIZE)
                    except socket.sslerror, e:
                        if e.args[0] != socket.SSL_ERROR_WANT_READ:
                            raise
                        logger.debug(
                            'SSL_ERROR_WANT_READ while sending data, retrying...')
                        continue
                    if not data:
                        logger.info('socket appears closed in read')
                        self.sock = None
                        self._current_response = None
                        if response is not None:
                            response._close()
                        # This if/elif ladder is a bit subtle,
                        # comments in each branch should help.
                        if response is not None and response.complete():
                            # Server responded completely and then

                            # after getting a really large response
                            # from the server.
                            logger.info(
                                'Connection appeared closed in read on first'
                                ' request loop iteration, will retry.')
                            reconnect('read')
                            continue
                        else:
                            # We didn't just send the first data hunk,
                            # and either have a partial response or no
                            # response at all. There's really nothing

                                'Connection appears closed after '
                                'some request data was written, but the '
                                'response was missing or incomplete!')
                    logger.debug('read %d bytes in request()', len(data))
                    if response is None:
                        response = self.response_class(r[0], self.timeout, method)
                    response._load_response(data)
                    # Jump to the next select() call so we load more
                    # data if the server is still sending us content.
                    continue
                except socket.error, e:

            # outgoing data
            if w and out:
                try:
                    if getattr(out, 'read', False):
                        data = out.read(OUTGOING_BUFFER_SIZE)
                        if not data:
                            continue
                        if len(data) < OUTGOING_BUFFER_SIZE:
                            if chunked:

                        # similar to selecting on a raw socket.
                        continue
                    elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
                          and not first):
                        raise
                    reconnect('write')
                    amt = self.sock.send(out)
                logger.debug('sent %d', amt)
                first = False
                # stash data we think we sent in case the socket breaks
                # when we read from it
                if was_first:
                    sent_data = out[:amt]
                if out is body:
                    body = out[amt:]
                else:
                    outgoing_headers = out[amt:]

        # close if the server response said to or responded before eating
        # the whole request
        if response is None:
            response = self.response_class(self.sock, self.timeout, method)
        complete = response.complete()
        data_left = bool(outgoing_headers or body)
        if data_left:
            logger.info('stopped sending request early, '
                         'will close the socket to be safe.')
            response.will_close = True

            # the socket
            self.sock = None
        self._current_response = response

    def getresponse(self):
        if self._current_response is None:
            raise httplib.ResponseNotReady()
        r = self._current_response
        while r.headers is None:
            if not r._select() and not r.complete():
                raise _readers.HTTPRemoteClosedError()
        if r.will_close:
            self.sock = None
            self._current_response = None

19182:fae47ecaa952

  * uses select() to block for incoming data
  * notices when the server responds early to a request
  * implements ssl inline instead of in a different class
"""

# Many functions in this file have too many arguments.
# pylint: disable=R0913

import cStringIO
import errno
import httplib
import logging
import rfc822

        if self._reader:
            return self._reader.done()

    def _close(self):
        if self._reader is not None:
            # We're a friend of the reader class here.
            # pylint: disable=W0212
            self._reader._close()

    def readline(self):
        """Read a single line from the response body.

            self._select()

        return ''.join(blocks)

    def read(self, length=None):
        """Read data from the response body."""
        # if length is None, unbounded read
        while (not self.complete()  # never select on a finished read
               and (not length  # unbounded, so we wait for complete()
                    or length > self._reader.available_data)):
            self._select()

        if self.complete() and self.will_close:
            self.sock.close()
        return r

    def _select(self):
        r, unused_write, unused_err = select.select(
            [self.sock], [], [], self._timeout)
        if not r:
            # socket was not readable. If the response is not
            # complete, raise a timeout.
            if not self.complete():
                logger.info('timed out with timeout of %s', self._timeout)

        # If the socket was readable and no data was read, that means
        # the socket was closed. Inform the reader (if any) so it can
        # raise an exception if this is an invalid situation.
        if not data:
            if self._reader:
                # We're a friend of the reader class here.
                # pylint: disable=W0212
                self._reader._close()
            return False
        else:
            self._load_response(data)
            return True

    # This method gets replaced by _load later, which confuses pylint.
    def _load_response(self, data): # pylint: disable=E0202
        # Being here implies we're not at the end of the headers yet,
        # since at the end of this method if headers were completely
        # loaded we replace this method with the load() method of the
        # reader we created.
        self.raw_response += data

        if self._end_headers not in self.raw_response or self.headers:
            return

        # handle 100-continue response
        hdrs, body = self.raw_response.split(self._end_headers, 1)
        unused_http_ver, status = hdrs.split(' ', 1)
        if status.startswith('100'):
            self.raw_response = body
            self.continued = True
            logger.debug('continue seen, setting body to %r', body)
            return

                self._reader = _readers.CloseIsEndReader()
                logger.debug('using a close-is-end reader')
                self.will_close = True

        if body:
            # We're a friend of the reader class here.
            # pylint: disable=W0212
            self._reader._load(body)
        logger.debug('headers complete')
        self.headers = headers
        # We're a friend of the reader class here.
        # pylint: disable=W0212
        self._load_response = self._reader._load


class HTTPConnection(object):
    """Connection to a single http server.

                        self._proxy_host, self._proxy_port)
            sock = socketutil.create_connection((self._proxy_host,
                                                 self._proxy_port))
            if self.ssl:
                # TODO proxy header support
                data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
                                                                self.port),
                                          {}, HTTP_VER_1_0)
                sock.send(data)
                sock.setblocking(0)
                r = self.response_class(sock, self.timeout, 'CONNECT')
                timeout_exc = HTTPTimeoutException(
                    'Timed out waiting for CONNECT response from proxy')
                while not r.complete():
                    try:
                        # We're a friend of the response class, so let
                        # us use the private attribute.
                        # pylint: disable=W0212
                        if not r._select():
                            if not r.complete():
                                raise timeout_exc
                    except HTTPTimeoutException:
                        # This raise/except pattern looks goofy, but

            if self._ssl_validator:
                self._ssl_validator(sock)
        sock.setblocking(0)
        self.sock = sock

    def _buildheaders(self, method, path, headers, http_ver):
        if self.ssl and self.port == 443 or self.port == 80:
            # default port for protocol, so leave it out
            hdrhost = self.host
        else:
            # include nonstandard port in header

                    self._current_response = None
                    return False
            return True
        return False

    def _reconnect(self, where):
        logger.info('reconnecting during %s', where)
        self.close()
        self._connect()

    def request(self, method, path, body=None, headers={},
                expect_continue=False):
        """Send a request to the server.

        For increased flexibility, this does not return the response

                chunked = True
            else:
                raise BadRequestData('body has no __len__() nor read()')

        self._connect()
        outgoing_headers = self._buildheaders(
            method, path, hdrs, self.http_version)
        response = None
        first = True

        while ((outgoing_headers or body)
               and not (response and response.complete())):
            select_timeout = self.timeout
            out = outgoing_headers or body

                    try:
                        data = r[0].recv(INCOMING_BUFFER_SIZE)
                    except socket.sslerror, e:
                        if e.args[0] != socket.SSL_ERROR_WANT_READ:
                            raise
                        logger.debug('SSL_ERROR_WANT_READ while sending '
                                     'data, retrying...')
                        continue
                    if not data:
                        logger.info('socket appears closed in read')
                        self.sock = None
                        self._current_response = None
                        if response is not None:
                            # We're a friend of the response class, so let
                            # us use the private attribute.
                            # pylint: disable=W0212
                            response._close()
                        # This if/elif ladder is a bit subtle,
                        # comments in each branch should help.
                        if response is not None and response.complete():
                            # Server responded completely and then

                            # after getting a really large response
                            # from the server.
                            logger.info(
                                'Connection appeared closed in read on first'
                                ' request loop iteration, will retry.')
                            self._reconnect('read')
                            continue
                        else:
                            # We didn't just send the first data hunk,
                            # and either have a partial response or no
                            # response at all. There's really nothing

                                'Connection appears closed after '
                                'some request data was written, but the '
                                'response was missing or incomplete!')
                    logger.debug('read %d bytes in request()', len(data))
                    if response is None:
                        response = self.response_class(
                            r[0], self.timeout, method)
                    # We're a friend of the response class, so let us
                    # use the private attribute.
                    # pylint: disable=W0212
                    response._load_response(data)
                    # Jump to the next select() call so we load more
                    # data if the server is still sending us content.
                    continue
                except socket.error, e:

            # outgoing data
            if w and out:
                try:
                    if getattr(out, 'read', False):
                        # pylint guesses the type of out incorrectly here
                        # pylint: disable=E1103
                        data = out.read(OUTGOING_BUFFER_SIZE)
                        if not data:
                            continue
                        if len(data) < OUTGOING_BUFFER_SIZE:
                            if chunked:

                        # similar to selecting on a raw socket.
                        continue
                    elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
                          and not first):
                        raise
                    self._reconnect('write')
                    amt = self.sock.send(out)
                logger.debug('sent %d', amt)
                first = False
                if out is body:
                    body = out[amt:]
                else:
                    outgoing_headers = out[amt:]

        # close if the server response said to or responded before eating
        # the whole request
        if response is None:
            response = self.response_class(self.sock, self.timeout, method)
        data_left = bool(outgoing_headers or body)
        if data_left:
            logger.info('stopped sending request early, '
                         'will close the socket to be safe.')
            response.will_close = True

            # the socket
            self.sock = None
        self._current_response = response

    def getresponse(self):
        """Returns the response to the most recent request."""
        if self._current_response is None:
            raise httplib.ResponseNotReady()
        r = self._current_response
        while r.headers is None:
            # We're a friend of the response class, so let us use the
            # private attribute.
            # pylint: disable=W0212
            if not r._select() and not r.complete():
                raise _readers.HTTPRemoteClosedError()
        if r.will_close:
            self.sock = None
            self._current_response = None