--- a/mercurial/httpclient/__init__.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/__init__.py Fri May 04 16:00:33 2012 -0500
@@ -45,6 +45,7 @@
import select
import socket
+import _readers
import socketutil
logger = logging.getLogger(__name__)
@@ -54,8 +55,6 @@
HTTP_VER_1_0 = 'HTTP/1.0'
HTTP_VER_1_1 = 'HTTP/1.1'
-_LEN_CLOSE_IS_END = -1
-
OUTGOING_BUFFER_SIZE = 1 << 15
INCOMING_BUFFER_SIZE = 1 << 20
@@ -83,23 +82,19 @@
The response will continue to load as available. If you need the
complete response before continuing, check the .complete() method.
"""
- def __init__(self, sock, timeout):
+ def __init__(self, sock, timeout, method):
self.sock = sock
+ self.method = method
self.raw_response = ''
- self._body = None
self._headers_len = 0
- self._content_len = 0
self.headers = None
self.will_close = False
self.status_line = ''
self.status = None
+ self.continued = False
self.http_version = None
self.reason = None
- self._chunked = False
- self._chunked_done = False
- self._chunked_until_next = 0
- self._chunked_skip_bytes = 0
- self._chunked_preloaded_block = None
+ self._reader = None
self._read_location = 0
self._eol = EOL
@@ -117,11 +112,12 @@
socket is closed, this will nearly always return False, even
in cases where all the data has actually been loaded.
"""
- if self._chunked:
- return self._chunked_done
- if self._content_len == _LEN_CLOSE_IS_END:
- return False
- return self._body is not None and len(self._body) >= self._content_len
+ if self._reader:
+ return self._reader.done()
+
+ def _close(self):
+ if self._reader is not None:
+ self._reader._close()
def readline(self):
"""Read a single line from the response body.
@@ -129,30 +125,34 @@
This may block until either a line ending is found or the
response is complete.
"""
- eol = self._body.find('\n', self._read_location)
- while eol == -1 and not self.complete():
+ # TODO: move this into the reader interface where it can be
+ # smarter (and probably avoid copies)
+ bytes = []
+ while not bytes:
+ try:
+ bytes = [self._reader.read(1)]
+ except _readers.ReadNotReady:
+ self._select()
+ while bytes[-1] != '\n' and not self.complete():
self._select()
- eol = self._body.find('\n', self._read_location)
- if eol != -1:
- eol += 1
- else:
- eol = len(self._body)
- data = self._body[self._read_location:eol]
- self._read_location = eol
- return data
+ bytes.append(self._reader.read(1))
+ if bytes[-1] != '\n':
+ next = self._reader.read(1)
+ while next and next != '\n':
+ bytes.append(next)
+ next = self._reader.read(1)
+ bytes.append(next)
+ return ''.join(bytes)
def read(self, length=None):
# if length is None, unbounded read
while (not self.complete() # never select on a finished read
and (not length # unbounded, so we wait for complete()
- or (self._read_location + length) > len(self._body))):
+ or length > self._reader.available_data)):
self._select()
if not length:
- length = len(self._body) - self._read_location
- elif len(self._body) < (self._read_location + length):
- length = len(self._body) - self._read_location
- r = self._body[self._read_location:self._read_location + length]
- self._read_location += len(r)
+ length = self._reader.available_data
+ r = self._reader.read(length)
if self.complete() and self.will_close:
self.sock.close()
return r
@@ -160,93 +160,35 @@
def _select(self):
r, _, _ = select.select([self.sock], [], [], self._timeout)
if not r:
- # socket was not readable. If the response is not complete
- # and we're not a _LEN_CLOSE_IS_END response, raise a timeout.
- # If we are a _LEN_CLOSE_IS_END response and we have no data,
- # raise a timeout.
- if not (self.complete() or
- (self._content_len == _LEN_CLOSE_IS_END and self._body)):
+ # socket was not readable. If the response is not
+ # complete, raise a timeout.
+ if not self.complete():
logger.info('timed out with timeout of %s', self._timeout)
raise HTTPTimeoutException('timeout reading data')
- logger.info('cl: %r body: %r', self._content_len, self._body)
try:
data = self.sock.recv(INCOMING_BUFFER_SIZE)
- # If the socket was readable and no data was read, that
- # means the socket was closed. If this isn't a
- # _CLOSE_IS_END socket, then something is wrong if we're
- # here (we shouldn't enter _select() if the response is
- # complete), so abort.
- if not data and self._content_len != _LEN_CLOSE_IS_END:
- raise HTTPRemoteClosedError(
- 'server appears to have closed the socket mid-response')
except socket.sslerror, e:
if e.args[0] != socket.SSL_ERROR_WANT_READ:
raise
logger.debug('SSL_WANT_READ in _select, should retry later')
return True
logger.debug('response read %d data during _select', len(data))
+ # If the socket was readable and no data was read, that means
+ # the socket was closed. Inform the reader (if any) so it can
+ # raise an exception if this is an invalid situation.
if not data:
- if self.headers and self._content_len == _LEN_CLOSE_IS_END:
- self._content_len = len(self._body)
+ if self._reader:
+ self._reader._close()
return False
else:
self._load_response(data)
return True
- def _chunked_parsedata(self, data):
- if self._chunked_preloaded_block:
- data = self._chunked_preloaded_block + data
- self._chunked_preloaded_block = None
- while data:
- logger.debug('looping with %d data remaining', len(data))
- # Slice out anything we should skip
- if self._chunked_skip_bytes:
- if len(data) <= self._chunked_skip_bytes:
- self._chunked_skip_bytes -= len(data)
- data = ''
- break
- else:
- data = data[self._chunked_skip_bytes:]
- self._chunked_skip_bytes = 0
-
- # determine how much is until the next chunk
- if self._chunked_until_next:
- amt = self._chunked_until_next
- logger.debug('reading remaining %d of existing chunk', amt)
- self._chunked_until_next = 0
- body = data
- else:
- try:
- amt, body = data.split(self._eol, 1)
- except ValueError:
- self._chunked_preloaded_block = data
- logger.debug('saving %r as a preloaded block for chunked',
- self._chunked_preloaded_block)
- return
- amt = int(amt, base=16)
- logger.debug('reading chunk of length %d', amt)
- if amt == 0:
- self._chunked_done = True
-
- # read through end of what we have or the chunk
- self._body += body[:amt]
- if len(body) >= amt:
- data = body[amt:]
- self._chunked_skip_bytes = len(self._eol)
- else:
- self._chunked_until_next = amt - len(body)
- self._chunked_skip_bytes = 0
- data = ''
-
def _load_response(self, data):
- if self._chunked:
- self._chunked_parsedata(data)
- return
- elif self._body is not None:
- self._body += data
- return
-
- # We haven't seen end of headers yet
+ # Being here implies we're not at the end of the headers yet,
+ # since at the end of this method if headers were completely
+ # loaded we replace this method with the load() method of the
+ # reader we created.
self.raw_response += data
# This is a bogus server with bad line endings
if self._eol not in self.raw_response:
@@ -270,6 +212,7 @@
http_ver, status = hdrs.split(' ', 1)
if status.startswith('100'):
self.raw_response = body
+ self.continued = True
logger.debug('continue seen, setting body to %r', body)
return
@@ -289,23 +232,46 @@
if self._eol != EOL:
hdrs = hdrs.replace(self._eol, '\r\n')
headers = rfc822.Message(cStringIO.StringIO(hdrs))
+ content_len = None
if HDR_CONTENT_LENGTH in headers:
- self._content_len = int(headers[HDR_CONTENT_LENGTH])
+ content_len = int(headers[HDR_CONTENT_LENGTH])
if self.http_version == HTTP_VER_1_0:
self.will_close = True
elif HDR_CONNECTION_CTRL in headers:
self.will_close = (
headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
- if self._content_len == 0:
- self._content_len = _LEN_CLOSE_IS_END
if (HDR_XFER_ENCODING in headers
and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
- self._body = ''
- self._chunked_parsedata(body)
- self._chunked = True
- if self._body is None:
- self._body = body
+ self._reader = _readers.ChunkedReader(self._eol)
+ logger.debug('using a chunked reader')
+ else:
+ # HEAD responses are forbidden from returning a body, and
+ # it's implausible for a CONNECT response to use
+ # close-is-end logic for an OK response.
+ if (self.method == 'HEAD' or
+ (self.method == 'CONNECT' and content_len is None)):
+ content_len = 0
+ if content_len is not None:
+ logger.debug('using a content-length reader with length %d',
+ content_len)
+ self._reader = _readers.ContentLengthReader(content_len)
+ else:
+ # Response body had no length specified and is not
+ # chunked, so the end of the body will only be
+ # identifiable by the termination of the socket by the
+ # server. My interpretation of the spec means that we
+ # are correct in hitting this case if
+ # transfer-encoding, content-length, and
+ # connection-control were left unspecified.
+ self._reader = _readers.CloseIsEndReader()
+ logger.debug('using a close-is-end reader')
+ self.will_close = True
+
+ if body:
+ self._reader._load(body)
+ logger.debug('headers complete')
self.headers = headers
+ self._load_response = self._reader._load
class HTTPConnection(object):
@@ -382,13 +348,14 @@
{}, HTTP_VER_1_0)
sock.send(data)
sock.setblocking(0)
- r = self.response_class(sock, self.timeout)
+ r = self.response_class(sock, self.timeout, 'CONNECT')
timeout_exc = HTTPTimeoutException(
'Timed out waiting for CONNECT response from proxy')
while not r.complete():
try:
if not r._select():
- raise timeout_exc
+ if not r.complete():
+ raise timeout_exc
except HTTPTimeoutException:
# This raise/except pattern looks goofy, but
# _select can raise the timeout as well as the
@@ -527,7 +494,7 @@
out = outgoing_headers or body
blocking_on_continue = False
if expect_continue and not outgoing_headers and not (
- response and response.headers):
+ response and (response.headers or response.continued)):
logger.info(
'waiting up to %s seconds for'
' continue response from server',
@@ -550,11 +517,6 @@
'server, optimistically sending request body')
else:
raise HTTPTimeoutException('timeout sending data')
- # TODO exceptional conditions with select? (what are those be?)
- # TODO if the response is loading, must we finish sending at all?
- #
- # Certainly not if it's going to close the connection and/or
- # the response is already done...I think.
was_first = first
# incoming data
@@ -572,11 +534,11 @@
logger.info('socket appears closed in read')
self.sock = None
self._current_response = None
+ if response is not None:
+ response._close()
# This if/elif ladder is a bit subtle,
# comments in each branch should help.
- if response is not None and (
- response.complete() or
- response._content_len == _LEN_CLOSE_IS_END):
+ if response is not None and response.complete():
# Server responded completely and then
# closed the socket. We should just shut
# things down and let the caller get their
@@ -605,7 +567,7 @@
'response was missing or incomplete!')
logger.debug('read %d bytes in request()', len(data))
if response is None:
- response = self.response_class(r[0], self.timeout)
+ response = self.response_class(r[0], self.timeout, method)
response._load_response(data)
# Jump to the next select() call so we load more
# data if the server is still sending us content.
@@ -613,10 +575,6 @@
except socket.error, e:
if e[0] != errno.EPIPE and not was_first:
raise
- if (response._content_len
- and response._content_len != _LEN_CLOSE_IS_END):
- outgoing_headers = sent_data + outgoing_headers
- reconnect('read')
# outgoing data
if w and out:
@@ -661,7 +619,7 @@
# close if the server response said to or responded before eating
# the whole request
if response is None:
- response = self.response_class(self.sock, self.timeout)
+ response = self.response_class(self.sock, self.timeout, method)
complete = response.complete()
data_left = bool(outgoing_headers or body)
if data_left:
@@ -679,7 +637,8 @@
raise httplib.ResponseNotReady()
r = self._current_response
while r.headers is None:
- r._select()
+ if not r._select() and not r.complete():
+ raise _readers.HTTPRemoteClosedError()
if r.will_close:
self.sock = None
self._current_response = None
@@ -705,7 +664,7 @@
class HTTPStateError(httplib.HTTPException):
"""Invalid internal state encountered."""
-
-class HTTPRemoteClosedError(httplib.HTTPException):
- """The server closed the remote socket in the middle of a response."""
+# Forward this exception type from _readers since it needs to be part
+# of the public API.
+HTTPRemoteClosedError = _readers.HTTPRemoteClosedError
# no-check-code
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/httpclient/_readers.py Fri May 04 16:00:33 2012 -0500
@@ -0,0 +1,195 @@
+# Copyright 2011, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Reader objects to abstract out different body response types.
+
+This module is package-private. It is not expected that these will
+have any clients outside of httpplus.
+"""
+
+import httplib
+import itertools
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ReadNotReady(Exception):
+ """Raised when read() is attempted but not enough data is loaded."""
+
+
+class HTTPRemoteClosedError(httplib.HTTPException):
+ """The server closed the remote socket in the middle of a response."""
+
+
+class AbstractReader(object):
+ """Abstract base class for response readers.
+
+ Subclasses must implement _load, and should implement _close if
+ it's not an error for the server to close their socket without
+ some termination condition being detected during _load.
+ """
+ def __init__(self):
+ self._finished = False
+ self._done_chunks = []
+
+ @property
+ def available_data(self):
+ return sum(map(len, self._done_chunks))
+
+ def done(self):
+ return self._finished
+
+ def read(self, amt):
+ if self.available_data < amt and not self._finished:
+ raise ReadNotReady()
+ need = [amt]
+ def pred(s):
+ needed = need[0] > 0
+ need[0] -= len(s)
+ return needed
+ blocks = list(itertools.takewhile(pred, self._done_chunks))
+ self._done_chunks = self._done_chunks[len(blocks):]
+ over_read = sum(map(len, blocks)) - amt
+ if over_read > 0 and blocks:
+ logger.debug('need to reinsert %d data into done chunks', over_read)
+ last = blocks[-1]
+ blocks[-1], reinsert = last[:-over_read], last[-over_read:]
+ self._done_chunks.insert(0, reinsert)
+ result = ''.join(blocks)
+ assert len(result) == amt or (self._finished and len(result) < amt)
+ return result
+
+ def _load(self, data): # pragma: no cover
+ """Subclasses must implement this.
+
+ As data is available to be read out of this object, it should
+ be placed into the _done_chunks list. Subclasses should not
+ rely on data remaining in _done_chunks forever, as it may be
+ reaped if the client is parsing data as it comes in.
+ """
+ raise NotImplementedError
+
+ def _close(self):
+ """Default implementation of close.
+
+ The default implementation assumes that the reader will mark
+ the response as finished on the _finished attribute once the
+ entire response body has been read. In the event that this is
+ not true, the subclass should override the implementation of
+ close (for example, close-is-end responses have to set
+ self._finished in the close handler.)
+ """
+ if not self._finished:
+ raise HTTPRemoteClosedError(
+ 'server appears to have closed the socket mid-response')
+
+
+class AbstractSimpleReader(AbstractReader):
+ """Abstract base class for simple readers that require no response decoding.
+
+ Examples of such responses are Connection: Close (close-is-end)
+ and responses that specify a content length.
+ """
+ def _load(self, data):
+ if data:
+ assert not self._finished, (
+ 'tried to add data (%r) to a closed reader!' % data)
+ logger.debug('%s read an addtional %d data', self.name, len(data))
+ self._done_chunks.append(data)
+
+
+class CloseIsEndReader(AbstractSimpleReader):
+ """Reader for responses that specify Connection: Close for length."""
+ name = 'close-is-end'
+
+ def _close(self):
+ logger.info('Marking close-is-end reader as closed.')
+ self._finished = True
+
+
+class ContentLengthReader(AbstractSimpleReader):
+ """Reader for responses that specify an exact content length."""
+ name = 'content-length'
+
+ def __init__(self, amount):
+ AbstractReader.__init__(self)
+ self._amount = amount
+ if amount == 0:
+ self._finished = True
+ self._amount_seen = 0
+
+ def _load(self, data):
+ AbstractSimpleReader._load(self, data)
+ self._amount_seen += len(data)
+ if self._amount_seen >= self._amount:
+ self._finished = True
+ logger.debug('content-length read complete')
+
+
+class ChunkedReader(AbstractReader):
+ """Reader for chunked transfer encoding responses."""
+ def __init__(self, eol):
+ AbstractReader.__init__(self)
+ self._eol = eol
+ self._leftover_skip_amt = 0
+ self._leftover_data = ''
+
+ def _load(self, data):
+ assert not self._finished, 'tried to add data to a closed reader!'
+ logger.debug('chunked read an addtional %d data', len(data))
+ position = 0
+ if self._leftover_data:
+ logger.debug('chunked reader trying to finish block from leftover data')
+ # TODO: avoid this string concatenation if possible
+ data = self._leftover_data + data
+ position = self._leftover_skip_amt
+ self._leftover_data = ''
+ self._leftover_skip_amt = 0
+ datalen = len(data)
+ while position < datalen:
+ split = data.find(self._eol, position)
+ if split == -1:
+ self._leftover_data = data
+ self._leftover_skip_amt = position
+ return
+ amt = int(data[position:split], base=16)
+ block_start = split + len(self._eol)
+ # If the whole data chunk plus the eol trailer hasn't
+ # loaded, we'll wait for the next load.
+ if block_start + amt + len(self._eol) > len(data):
+ self._leftover_data = data
+ self._leftover_skip_amt = position
+ return
+ if amt == 0:
+ self._finished = True
+ logger.debug('closing chunked redaer due to chunk of length 0')
+ return
+ self._done_chunks.append(data[block_start:block_start + amt])
+ position = block_start + amt + len(self._eol)
+# no-check-code
--- a/mercurial/httpclient/tests/simple_http_test.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/tests/simple_http_test.py Fri May 04 16:00:33 2012 -0500
@@ -29,7 +29,7 @@
import socket
import unittest
-import http
+import httpplus
# relative import to ease embedding the library
import util
@@ -38,7 +38,7 @@
class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
def _run_simple_test(self, host, server_data, expected_req, expected_data):
- con = http.HTTPConnection(host)
+ con = httpplus.HTTPConnection(host)
con._connect()
con.sock.data = server_data
con.request('GET', '/')
@@ -47,9 +47,9 @@
self.assertEqual(expected_data, con.getresponse().read())
def test_broken_data_obj(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
- self.assertRaises(http.BadRequestData,
+ self.assertRaises(httpplus.BadRequestData,
con.request, 'POST', '/', body=1)
def test_no_keepalive_http_1_0(self):
@@ -74,7 +74,7 @@
fncache
dotencode
"""
- con = http.HTTPConnection('localhost:9999')
+ con = httpplus.HTTPConnection('localhost:9999')
con._connect()
con.sock.data = [expected_response_headers, expected_response_body]
con.request('GET', '/remote/.hg/requires',
@@ -95,7 +95,7 @@
self.assert_(resp.sock.closed)
def test_multiline_header(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
con.sock.data = ['HTTP/1.1 200 OK\r\n',
'Server: BogusServer 1.0\r\n',
@@ -122,7 +122,7 @@
self.assertEqual(con.sock.closed, False)
def testSimpleRequest(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
con.sock.data = ['HTTP/1.1 200 OK\r\n',
'Server: BogusServer 1.0\r\n',
@@ -149,12 +149,13 @@
resp.headers.getheaders('server'))
def testHeaderlessResponse(self):
- con = http.HTTPConnection('1.2.3.4', use_ssl=False)
+ con = httpplus.HTTPConnection('1.2.3.4', use_ssl=False)
con._connect()
con.sock.data = ['HTTP/1.1 200 OK\r\n',
'\r\n'
'1234567890'
]
+ con.sock.close_on_empty = True
con.request('GET', '/')
expected_req = ('GET / HTTP/1.1\r\n'
@@ -169,7 +170,30 @@
self.assertEqual(resp.status, 200)
def testReadline(self):
- con = http.HTTPConnection('1.2.3.4')
+ con = httpplus.HTTPConnection('1.2.3.4')
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Connection: Close\r\n',
+ '\r\n'
+ '1\n2\nabcdefg\n4\n5']
+ con.sock.close_on_empty = True
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ con.request('GET', '/')
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ r = con.getresponse()
+ for expected in ['1\n', '2\n', 'abcdefg\n', '4\n', '5']:
+ actual = r.readline()
+ self.assertEqual(expected, actual,
+ 'Expected %r, got %r' % (expected, actual))
+
+ def testReadlineTrickle(self):
+ con = httpplus.HTTPConnection('1.2.3.4')
con._connect()
# make sure it trickles in one byte at a time
# so that we touch all the cases in readline
@@ -179,6 +203,7 @@
'Connection: Close\r\n',
'\r\n'
'1\n2\nabcdefg\n4\n5']))
+ con.sock.close_on_empty = True
expected_req = ('GET / HTTP/1.1\r\n'
'Host: 1.2.3.4\r\n'
@@ -193,6 +218,59 @@
self.assertEqual(expected, actual,
'Expected %r, got %r' % (expected, actual))
+ def testVariousReads(self):
+ con = httpplus.HTTPConnection('1.2.3.4')
+ con._connect()
+ # make sure it trickles in one byte at a time
+ # so that we touch all the cases in readline
+ con.sock.data = list(''.join(
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Connection: Close\r\n',
+ '\r\n'
+ '1\n2',
+ '\na', 'bc',
+ 'defg\n4\n5']))
+ con.sock.close_on_empty = True
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ con.request('GET', '/')
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ r = con.getresponse()
+ for read_amt, expect in [(1, '1'), (1, '\n'),
+ (4, '2\nab'),
+ ('line', 'cdefg\n'),
+ (None, '4\n5')]:
+ if read_amt == 'line':
+ self.assertEqual(expect, r.readline())
+ else:
+ self.assertEqual(expect, r.read(read_amt))
+
+ def testZeroLengthBody(self):
+ con = httpplus.HTTPConnection('1.2.3.4')
+ con._connect()
+ # make sure it trickles in one byte at a time
+ # so that we touch all the cases in readline
+ con.sock.data = list(''.join(
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-length: 0\r\n',
+ '\r\n']))
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ con.request('GET', '/')
+ self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+ self.assertEqual(expected_req, con.sock.sent)
+ r = con.getresponse()
+ self.assertEqual('', r.read())
+
def testIPv6(self):
self._run_simple_test('[::1]:8221',
['HTTP/1.1 200 OK\r\n',
@@ -226,7 +304,7 @@
'1234567890')
def testEarlyContinueResponse(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.data = ['HTTP/1.1 403 Forbidden\r\n',
@@ -240,8 +318,23 @@
self.assertEqual("You can't do that.", con.getresponse().read())
self.assertEqual(sock.closed, True)
+ def testEarlyContinueResponseNoContentLength(self):
+ con = httpplus.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ '\r\n'
+ "You can't do that."]
+ sock.close_on_empty = True
+ expected_req = self.doPost(con, expect_body=False)
+ self.assertEqual(('1.2.3.4', 80), sock.sa)
+ self.assertStringEqual(expected_req, sock.sent)
+ self.assertEqual("You can't do that.", con.getresponse().read())
+ self.assertEqual(sock.closed, True)
+
def testDeniedAfterContinueTimeoutExpires(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.data = ['HTTP/1.1 403 Forbidden\r\n',
@@ -269,7 +362,7 @@
self.assertEqual(sock.closed, True)
def testPostData(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.read_wait_sentinel = 'POST data'
@@ -286,7 +379,7 @@
self.assertEqual(sock.closed, False)
def testServerWithoutContinue(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.read_wait_sentinel = 'POST data'
@@ -302,7 +395,7 @@
self.assertEqual(sock.closed, False)
def testServerWithSlowContinue(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.read_wait_sentinel = 'POST data'
@@ -321,7 +414,7 @@
self.assertEqual(sock.closed, False)
def testSlowConnection(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
# simulate one byte arriving at a time, to check for various
# corner cases
@@ -340,12 +433,26 @@
self.assertEqual(expected_req, con.sock.sent)
self.assertEqual('1234567890', con.getresponse().read())
+ def testCloseAfterNotAllOfHeaders(self):
+ con = httpplus.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: NO CARRIER']
+ con.sock.close_on_empty = True
+ con.request('GET', '/')
+ self.assertRaises(httpplus.HTTPRemoteClosedError,
+ con.getresponse)
+
+ expected_req = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
def testTimeout(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
con.sock.data = []
con.request('GET', '/')
- self.assertRaises(http.HTTPTimeoutException,
+ self.assertRaises(httpplus.HTTPTimeoutException,
con.getresponse)
expected_req = ('GET / HTTP/1.1\r\n'
@@ -370,7 +477,7 @@
return s
socket.socket = closingsocket
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
con.request('GET', '/')
r1 = con.getresponse()
@@ -381,7 +488,7 @@
self.assertEqual(2, len(sockets))
def test_server_closes_before_end_of_body(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
s = con.sock
s.data = ['HTTP/1.1 200 OK\r\n',
@@ -393,9 +500,9 @@
s.close_on_empty = True
con.request('GET', '/')
r1 = con.getresponse()
- self.assertRaises(http.HTTPRemoteClosedError, r1.read)
+ self.assertRaises(httpplus.HTTPRemoteClosedError, r1.read)
def test_no_response_raises_response_not_ready(self):
- con = http.HTTPConnection('foo')
- self.assertRaises(http.httplib.ResponseNotReady, con.getresponse)
+ con = httpplus.HTTPConnection('foo')
+ self.assertRaises(httpplus.httplib.ResponseNotReady, con.getresponse)
# no-check-code
--- a/mercurial/httpclient/tests/test_bogus_responses.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/tests/test_bogus_responses.py Fri May 04 16:00:33 2012 -0500
@@ -34,7 +34,7 @@
"""
import unittest
-import http
+import httpplus
# relative import to ease embedding the library
import util
@@ -43,7 +43,7 @@
class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
def bogusEOL(self, eol):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
con.sock.data = ['HTTP/1.1 200 OK%s' % eol,
'Server: BogusServer 1.0%s' % eol,
--- a/mercurial/httpclient/tests/test_chunked_transfer.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/tests/test_chunked_transfer.py Fri May 04 16:00:33 2012 -0500
@@ -29,7 +29,7 @@
import cStringIO
import unittest
-import http
+import httpplus
# relative import to ease embedding the library
import util
@@ -50,7 +50,7 @@
class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase):
def testChunkedUpload(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.read_wait_sentinel = '0\r\n\r\n'
@@ -77,7 +77,7 @@
self.assertEqual(sock.closed, False)
def testChunkedDownload(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.data = ['HTTP/1.1 200 OK\r\n',
@@ -85,14 +85,31 @@
'transfer-encoding: chunked',
'\r\n\r\n',
chunkedblock('hi '),
- chunkedblock('there'),
+ ] + list(chunkedblock('there')) + [
chunkedblock(''),
]
con.request('GET', '/')
self.assertStringEqual('hi there', con.getresponse().read())
+ def testChunkedDownloadOddReadBoundaries(self):
+ con = httpplus.HTTPConnection('1.2.3.4:80')
+ con._connect()
+ sock = con.sock
+ sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'transfer-encoding: chunked',
+ '\r\n\r\n',
+ chunkedblock('hi '),
+ ] + list(chunkedblock('there')) + [
+ chunkedblock(''),
+ ]
+ con.request('GET', '/')
+ resp = con.getresponse()
+ for amt, expect in [(1, 'h'), (5, 'i the'), (100, 're')]:
+ self.assertEqual(expect, resp.read(amt))
+
def testChunkedDownloadBadEOL(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.data = ['HTTP/1.1 200 OK\n',
@@ -107,7 +124,7 @@
self.assertStringEqual('hi there', con.getresponse().read())
def testChunkedDownloadPartialChunkBadEOL(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.data = ['HTTP/1.1 200 OK\n',
@@ -122,7 +139,7 @@
con.getresponse().read())
def testChunkedDownloadPartialChunk(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
sock.data = ['HTTP/1.1 200 OK\r\n',
@@ -136,7 +153,7 @@
con.getresponse().read())
def testChunkedDownloadEarlyHangup(self):
- con = http.HTTPConnection('1.2.3.4:80')
+ con = httpplus.HTTPConnection('1.2.3.4:80')
con._connect()
sock = con.sock
broken = chunkedblock('hi'*20)[:-1]
@@ -149,5 +166,5 @@
sock.close_on_empty = True
con.request('GET', '/')
resp = con.getresponse()
- self.assertRaises(http.HTTPRemoteClosedError, resp.read)
+ self.assertRaises(httpplus.HTTPRemoteClosedError, resp.read)
# no-check-code
--- a/mercurial/httpclient/tests/test_proxy_support.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/tests/test_proxy_support.py Fri May 04 16:00:33 2012 -0500
@@ -29,13 +29,13 @@
import unittest
import socket
-import http
+import httpplus
# relative import to ease embedding the library
import util
-def make_preloaded_socket(data):
+def make_preloaded_socket(data, close=False):
"""Make a socket pre-loaded with data so it can be read during connect.
Useful for https proxy tests because we have to read from the
@@ -44,6 +44,7 @@
def s(*args, **kwargs):
sock = util.MockSocket(*args, **kwargs)
sock.early_data = data[:]
+ sock.close_on_empty = close
return sock
return s
@@ -51,7 +52,7 @@
class ProxyHttpTest(util.HttpTestBase, unittest.TestCase):
def _run_simple_test(self, host, server_data, expected_req, expected_data):
- con = http.HTTPConnection(host)
+ con = httpplus.HTTPConnection(host)
con._connect()
con.sock.data = server_data
con.request('GET', '/')
@@ -60,7 +61,7 @@
self.assertEqual(expected_data, con.getresponse().read())
def testSimpleRequest(self):
- con = http.HTTPConnection('1.2.3.4:80',
+ con = httpplus.HTTPConnection('1.2.3.4:80',
proxy_hostport=('magicproxy', 4242))
con._connect()
con.sock.data = ['HTTP/1.1 200 OK\r\n',
@@ -88,7 +89,7 @@
resp.headers.getheaders('server'))
def testSSLRequest(self):
- con = http.HTTPConnection('1.2.3.4:443',
+ con = httpplus.HTTPConnection('1.2.3.4:443',
proxy_hostport=('magicproxy', 4242))
socket.socket = make_preloaded_socket(
['HTTP/1.1 200 OK\r\n',
@@ -124,12 +125,47 @@
self.assertEqual(['BogusServer 1.0'],
resp.headers.getheaders('server'))
- def testSSLProxyFailure(self):
- con = http.HTTPConnection('1.2.3.4:443',
+ def testSSLRequestNoConnectBody(self):
+ con = httpplus.HTTPConnection('1.2.3.4:443',
proxy_hostport=('magicproxy', 4242))
socket.socket = make_preloaded_socket(
- ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n'])
- self.assertRaises(http.HTTPProxyConnectFailedException, con._connect)
- self.assertRaises(http.HTTPProxyConnectFailedException,
+ ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ '\r\n'])
+ con._connect()
+ con.sock.data = ['HTTP/1.1 200 OK\r\n',
+ 'Server: BogusServer 1.0\r\n',
+ 'Content-Length: 10\r\n',
+ '\r\n'
+ '1234567890'
+ ]
+ connect_sent = con.sock.sent
+ con.sock.sent = ''
+ con.request('GET', '/')
+
+ expected_connect = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n'
+ '\r\n')
+ expected_request = ('GET / HTTP/1.1\r\n'
+ 'Host: 1.2.3.4\r\n'
+ 'accept-encoding: identity\r\n\r\n')
+
+ self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+ self.assertStringEqual(expected_connect, connect_sent)
+ self.assertStringEqual(expected_request, con.sock.sent)
+ resp = con.getresponse()
+ self.assertEqual(resp.status, 200)
+ self.assertEqual('1234567890', resp.read())
+ self.assertEqual(['BogusServer 1.0'],
+ resp.headers.getheaders('server'))
+
+ def testSSLProxyFailure(self):
+ con = httpplus.HTTPConnection('1.2.3.4:443',
+ proxy_hostport=('magicproxy', 4242))
+ socket.socket = make_preloaded_socket(
+ ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n'], close=True)
+ self.assertRaises(httpplus.HTTPProxyConnectFailedException, con._connect)
+ self.assertRaises(httpplus.HTTPProxyConnectFailedException,
con.request, 'GET', '/')
# no-check-code
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/httpclient/tests/test_readers.py Fri May 04 16:00:33 2012 -0500
@@ -0,0 +1,70 @@
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+from httpplus import _readers
+
+def chunkedblock(x, eol='\r\n'):
+ r"""Make a chunked transfer-encoding block.
+
+ >>> chunkedblock('hi')
+ '2\r\nhi\r\n'
+ >>> chunkedblock('hi' * 10)
+ '14\r\nhihihihihihihihihihi\r\n'
+ >>> chunkedblock('hi', eol='\n')
+ '2\nhi\n'
+ """
+ return ''.join((hex(len(x))[2:], eol, x, eol))
+
+corpus = 'foo\r\nbar\r\nbaz\r\n'
+
+
+class ChunkedReaderTest(unittest.TestCase):
+ def test_many_block_boundaries(self):
+ for step in xrange(1, len(corpus)):
+ data = ''.join(chunkedblock(corpus[start:start+step]) for
+ start in xrange(0, len(corpus), step))
+ for istep in xrange(1, len(data)):
+ rdr = _readers.ChunkedReader('\r\n')
+ print 'step', step, 'load', istep
+ for start in xrange(0, len(data), istep):
+ rdr._load(data[start:start+istep])
+ rdr._load(chunkedblock(''))
+ self.assertEqual(corpus, rdr.read(len(corpus) + 1))
+
+ def test_small_chunk_blocks_large_wire_blocks(self):
+ data = ''.join(map(chunkedblock, corpus)) + chunkedblock('')
+ rdr = _readers.ChunkedReader('\r\n')
+ for start in xrange(0, len(data), 4):
+ d = data[start:start + 4]
+ if d:
+ rdr._load(d)
+ self.assertEqual(corpus, rdr.read(len(corpus)+100))
+# no-check-code
--- a/mercurial/httpclient/tests/test_ssl.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/tests/test_ssl.py Fri May 04 16:00:33 2012 -0500
@@ -28,7 +28,7 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import unittest
-import http
+import httpplus
# relative import to ease embedding the library
import util
@@ -37,7 +37,7 @@
class HttpSslTest(util.HttpTestBase, unittest.TestCase):
def testSslRereadRequired(self):
- con = http.HTTPConnection('1.2.3.4:443')
+ con = httpplus.HTTPConnection('1.2.3.4:443')
con._connect()
# extend the list instead of assign because of how
# MockSSLSocket works.
@@ -66,7 +66,7 @@
resp.headers.getheaders('server'))
def testSslRereadInEarlyResponse(self):
- con = http.HTTPConnection('1.2.3.4:443')
+ con = httpplus.HTTPConnection('1.2.3.4:443')
con._connect()
con.sock.early_data = ['HTTP/1.1 200 OK\r\n',
'Server: BogusServer 1.0\r\n',
--- a/mercurial/httpclient/tests/util.py Sat May 12 00:06:11 2012 +0200
+++ b/mercurial/httpclient/tests/util.py Fri May 04 16:00:33 2012 -0500
@@ -29,7 +29,7 @@
import difflib
import socket
-import http
+import httpplus
class MockSocket(object):
@@ -57,7 +57,7 @@
self.remote_closed = self.closed = False
self.close_on_empty = False
self.sent = ''
- self.read_wait_sentinel = http._END_HEADERS
+ self.read_wait_sentinel = httpplus._END_HEADERS
def close(self):
self.closed = True
@@ -86,7 +86,7 @@
@property
def ready_for_read(self):
- return ((self.early_data and http._END_HEADERS in self.sent)
+ return ((self.early_data and httpplus._END_HEADERS in self.sent)
or (self.read_wait_sentinel in self.sent and self.data)
or self.closed or self.remote_closed)
@@ -132,7 +132,7 @@
def mocksslwrap(sock, keyfile=None, certfile=None,
- server_side=False, cert_reqs=http.socketutil.CERT_NONE,
+ server_side=False, cert_reqs=httpplus.socketutil.CERT_NONE,
ssl_version=None, ca_certs=None,
do_handshake_on_connect=True,
suppress_ragged_eofs=True):
@@ -156,16 +156,16 @@
self.orig_getaddrinfo = socket.getaddrinfo
socket.getaddrinfo = mockgetaddrinfo
- self.orig_select = http.select.select
- http.select.select = mockselect
+ self.orig_select = httpplus.select.select
+ httpplus.select.select = mockselect
- self.orig_sslwrap = http.socketutil.wrap_socket
- http.socketutil.wrap_socket = mocksslwrap
+ self.orig_sslwrap = httpplus.socketutil.wrap_socket
+ httpplus.socketutil.wrap_socket = mocksslwrap
def tearDown(self):
socket.socket = self.orig_socket
- http.select.select = self.orig_select
- http.socketutil.wrap_socket = self.orig_sslwrap
+ httpplus.select.select = self.orig_select
+ httpplus.socketutil.wrap_socket = self.orig_sslwrap
socket.getaddrinfo = self.orig_getaddrinfo
def assertStringEqual(self, l, r):