comparison mercurial/httpclient/__init__.py @ 27601:1ad9da968a2e

httpclient: update to 938f2107d6e2 of httpplus

This enhances proxy support in httpclient a little bit, though I don't
know that we used that functionality at all. It also switches httpplus
to using absolute_import.

27600:cfb26146a8cd

  * supports keepalives natively
  * uses select() to block for incoming data
  * notices when the server responds early to a request
  * implements ssl inline instead of in a different class
"""

# Many functions in this file have too many arguments.
# pylint: disable=R0913

import cStringIO

import logging
import rfc822
import select
import socket

import _readers
import socketutil

logger = logging.getLogger(__name__)

__all__ = ['HTTPConnection', 'HTTPResponse']

    def _close(self):
        if self._reader is not None:
            # We're a friend of the reader class here.
            # pylint: disable=W0212
            self._reader._close()

    def readline(self):
        """Read a single line from the response body.

        This may block until either a line ending is found or the

        self.headers = headers
        # We're a friend of the reader class here.
        # pylint: disable=W0212
        self._load_response = self._reader._load


class HTTPConnection(object):
    """Connection to a single http server.

    Supports 100-continue and keepalives natively. Uses select() for

    response_class = HTTPResponse

    def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
                 timeout=TIMEOUT_DEFAULT,
                 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
                 proxy_hostport=None, ssl_wrap_socket=None, **ssl_opts):
        """Create a new HTTPConnection.

        Args:
          host: The host to which we'll connect.
          port: Optional. The port over which we'll connect. Default 80 for

          timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
          continue_timeout: Optional. Timeout for waiting on an expected
                   "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
          proxy_hostport: Optional. Tuple of (host, port) to use as an http
                       proxy for the connection. Default is to not use a proxy.
          ssl_wrap_socket: Optional function to use for wrapping
            sockets. If unspecified, the one from the ssl module will
            be used if available, or something that's compatible with
            it if on a Python older than 2.6.

            use_ssl = False
            port = 80
        elif use_ssl is None:
            use_ssl = (port == 443)
        elif port is None:
            if use_ssl:
                port = 443
            else:
                port = 80
        self.port = port
        if use_ssl and not socketutil.have_ssl:
            raise Exception('ssl requested but unavailable on this Python')
        self.ssl = use_ssl
        self.ssl_opts = ssl_opts

        self.sock = None
        self._current_response = None
        self._current_response_taken = False
        if proxy_hostport is None:
            self._proxy_host = self._proxy_port = None
        else:
            self._proxy_host, self._proxy_port = proxy_hostport

        self.timeout = timeout
        self.continue_timeout = continue_timeout

    def _connect(self):
        """Connect to the host and port specified in __init__."""
        if self.sock:
            return
        if self._proxy_host is not None:
            logger.info('Connecting to http proxy %s:%s',
                        self._proxy_host, self._proxy_port)
            sock = socketutil.create_connection((self._proxy_host,
                                                 self._proxy_port))
            if self.ssl:
                # TODO proxy header support
                data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
                                                                self.port),
                                          {}, HTTP_VER_1_0)
                sock.send(data)
                sock.setblocking(0)
                r = self.response_class(sock, self.timeout, 'CONNECT')
                timeout_exc = HTTPTimeoutException(
                    'Timed out waiting for CONNECT response from proxy')

                    self._current_response = None
                    return False
            return True
        return False

    def _reconnect(self, where):
        logger.info('reconnecting during %s', where)
        self.close()
        self._connect()

    def request(self, method, path, body=None, headers={},
                expect_continue=False):
        """Send a request to the server.

                'current response is read!')
        self._current_response_taken = False

        logger.info('sending %s request for %s to %s on port %s',
                    method, path, self.host, self.port)
        hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
        if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
            expect_continue = True
        elif expect_continue:
            hdrs['expect'] = ('Expect', '100-Continue')

        chunked = False
        if body and HDR_CONTENT_LENGTH not in hdrs:
            if getattr(body, '__len__', False):
                hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))

        # If we're reusing the underlying socket, there are some
        # conditions where we'll want to retry, so make a note of the
        # state of self.sock
        fresh_socket = self.sock is None
        self._connect()
        outgoing_headers = self._buildheaders(
            method, path, hdrs, self.http_version)
        response = None
        first = True

                            # after getting a really large response
                            # from the server.
                            logger.info(
                                'Connection appeared closed in read on first'
                                ' request loop iteration, will retry.')
                            self._reconnect('read')
                            continue
                        else:
                            # We didn't just send the first data hunk,
                            # and either have a partial response or no
                            # response at all. There's really nothing

                    if e[0] == errno.EWOULDBLOCK or e[0] == errno.EAGAIN:
                        continue
                    elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
                          and not first):
                        raise
                    self._reconnect('write')
                    amt = self.sock.send(out)
                logger.debug('sent %d', amt)
                first = False
                if out is body:
                    body = out[amt:]

                if not response._select():
                    # This means the response failed to get any response
                    # data at all, and in all probability the socket was
                    # closed before the server even saw our request. Try
                    # the request again on a fresh socket.
                    logging.debug('response._select() failed during request().'
                                  ' Assuming request needs to be retried.')
                    self.sock = None
                    # Call this method explicitly to re-try the
                    # request. We don't use self.request() because
                    # some tools (notably Mercurial) expect to be able
                    # to subclass and redefine request(), and they

27601:1ad9da968a2e

  * supports keepalives natively
  * uses select() to block for incoming data
  * notices when the server responds early to a request
  * implements ssl inline instead of in a different class
"""
from __future__ import absolute_import

# Many functions in this file have too many arguments.
# pylint: disable=R0913

import cStringIO

import logging
import rfc822
import select
import socket

from . import (
    _readers,
    socketutil,
    )

logger = logging.getLogger(__name__)

__all__ = ['HTTPConnection', 'HTTPResponse']

    def _close(self):
        if self._reader is not None:
            # We're a friend of the reader class here.
            # pylint: disable=W0212
            self._reader._close()

    def getheader(self, header, default=None):
        return self.headers.getheader(header, default=default)

    def getheaders(self):
        return self.headers.items()

    def readline(self):
        """Read a single line from the response body.

        This may block until either a line ending is found or the

        self.headers = headers
        # We're a friend of the reader class here.
        # pylint: disable=W0212
        self._load_response = self._reader._load

def _foldheaders(headers):
    """Given some headers, rework them so we can safely overwrite values.

    >>> _foldheaders({'Accept-Encoding': 'wat'})
    {'accept-encoding': ('Accept-Encoding', 'wat')}
    """
    return dict((k.lower(), (k, v)) for k, v in headers.iteritems())


class HTTPConnection(object):
    """Connection to a single http server.

    Supports 100-continue and keepalives natively. Uses select() for

    response_class = HTTPResponse

    def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
                 timeout=TIMEOUT_DEFAULT,
                 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
                 proxy_hostport=None, proxy_headers=None,
                 ssl_wrap_socket=None, **ssl_opts):
        """Create a new HTTPConnection.

        Args:
          host: The host to which we'll connect.
          port: Optional. The port over which we'll connect. Default 80 for

          timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
          continue_timeout: Optional. Timeout for waiting on an expected
                   "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
          proxy_hostport: Optional. Tuple of (host, port) to use as an http
                       proxy for the connection. Default is to not use a proxy.
          proxy_headers: Optional dict of header keys and values to send to
                         a proxy when using CONNECT. For compatibility with
                         httplib, the Proxy-Authorization header may be
                         specified in headers for request(), which will clobber
                         any such header specified here if specified. Providing
                         this option and not proxy_hostport will raise an
                         ValueError.
          ssl_wrap_socket: Optional function to use for wrapping
            sockets. If unspecified, the one from the ssl module will
            be used if available, or something that's compatible with
            it if on a Python older than 2.6.

            use_ssl = False
            port = 80
        elif use_ssl is None:
            use_ssl = (port == 443)
        elif port is None:
            port = (use_ssl and 443 or 80)
        self.port = port
        if use_ssl and not socketutil.have_ssl:
            raise Exception('ssl requested but unavailable on this Python')
        self.ssl = use_ssl
        self.ssl_opts = ssl_opts

        self.sock = None
        self._current_response = None
        self._current_response_taken = False
        if proxy_hostport is None:
            self._proxy_host = self._proxy_port = None
            if proxy_headers:
                raise ValueError(
                    'proxy_headers may not be specified unless '
                    'proxy_hostport is also specified.')
            else:
                self._proxy_headers = {}
        else:
            self._proxy_host, self._proxy_port = proxy_hostport
            self._proxy_headers = _foldheaders(proxy_headers or {})

        self.timeout = timeout
        self.continue_timeout = continue_timeout

    def _connect(self, proxy_headers):
        """Connect to the host and port specified in __init__."""
        if self.sock:
            return
        if self._proxy_host is not None:
            logger.info('Connecting to http proxy %s:%s',
                        self._proxy_host, self._proxy_port)
            sock = socketutil.create_connection((self._proxy_host,
                                                 self._proxy_port))
            if self.ssl:
                data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
                                                                self.port),
                                          proxy_headers, HTTP_VER_1_0)
                sock.send(data)
                sock.setblocking(0)
                r = self.response_class(sock, self.timeout, 'CONNECT')
                timeout_exc = HTTPTimeoutException(
                    'Timed out waiting for CONNECT response from proxy')

                    self._current_response = None
                    return False
            return True
        return False

    def _reconnect(self, where, pheaders):
        logger.info('reconnecting during %s', where)
        self.close()
        self._connect(pheaders)

    def request(self, method, path, body=None, headers={},
                expect_continue=False):
        """Send a request to the server.

                'current response is read!')
        self._current_response_taken = False

        logger.info('sending %s request for %s to %s on port %s',
                    method, path, self.host, self.port)
        hdrs = _foldheaders(headers)
        if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
            expect_continue = True
        elif expect_continue:
            hdrs['expect'] = ('Expect', '100-Continue')
        # httplib compatibility: if the user specified a
        # proxy-authorization header, that's actually intended for a
        # proxy CONNECT action, not the real request, but only if
        # we're going to use a proxy.
        pheaders = dict(self._proxy_headers)
        if self._proxy_host and self.ssl:
            pa = hdrs.pop('proxy-authorization', None)
            if pa is not None:
                pheaders['proxy-authorization'] = pa

        chunked = False
        if body and HDR_CONTENT_LENGTH not in hdrs:
            if getattr(body, '__len__', False):
                hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))

        # If we're reusing the underlying socket, there are some
        # conditions where we'll want to retry, so make a note of the
        # state of self.sock
        fresh_socket = self.sock is None
        self._connect(pheaders)
        outgoing_headers = self._buildheaders(
            method, path, hdrs, self.http_version)
        response = None
        first = True

                            # after getting a really large response
                            # from the server.
                            logger.info(
                                'Connection appeared closed in read on first'
                                ' request loop iteration, will retry.')
                            self._reconnect('read', pheaders)
                            continue
                        else:
                            # We didn't just send the first data hunk,
                            # and either have a partial response or no
                            # response at all. There's really nothing

                    if e[0] == errno.EWOULDBLOCK or e[0] == errno.EAGAIN:
                        continue
                    elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
                          and not first):
                        raise
                    self._reconnect('write', pheaders)
                    amt = self.sock.send(out)
                logger.debug('sent %d', amt)
                first = False
                if out is body:
                    body = out[amt:]

                if not response._select():
                    # This means the response failed to get any response
                    # data at all, and in all probability the socket was
                    # closed before the server even saw our request. Try
                    # the request again on a fresh socket.
                    logger.debug('response._select() failed during request().'
                                 ' Assuming request needs to be retried.')
                    self.sock = None
                    # Call this method explicitly to re-try the
                    # request. We don't use self.request() because
                    # some tools (notably Mercurial) expect to be able
                    # to subclass and redefine request(), and they