Mercurial > hg-stable
changeset 16643:24dbef11f477
httpclient: update to revision 892730fe7f46 of httpplus
| author | Augie Fackler <raf@durin42.com> |
|---|---|
| date | Fri, 04 May 2012 16:00:33 -0500 |
| parents | 5cf18921bb7b |
| children | 98a9266db803 |
| files | mercurial/httpclient/__init__.py mercurial/httpclient/_readers.py mercurial/httpclient/tests/simple_http_test.py mercurial/httpclient/tests/test_bogus_responses.py mercurial/httpclient/tests/test_chunked_transfer.py mercurial/httpclient/tests/test_proxy_support.py mercurial/httpclient/tests/test_readers.py mercurial/httpclient/tests/test_ssl.py mercurial/httpclient/tests/util.py |
| diffstat | 9 files changed, 570 insertions(+), 186 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/httpclient/__init__.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/__init__.py Fri May 04 16:00:33 2012 -0500 @@ -45,6 +45,7 @@ import select import socket +import _readers import socketutil logger = logging.getLogger(__name__) @@ -54,8 +55,6 @@ HTTP_VER_1_0 = 'HTTP/1.0' HTTP_VER_1_1 = 'HTTP/1.1' -_LEN_CLOSE_IS_END = -1 - OUTGOING_BUFFER_SIZE = 1 << 15 INCOMING_BUFFER_SIZE = 1 << 20 @@ -83,23 +82,19 @@ The response will continue to load as available. If you need the complete response before continuing, check the .complete() method. """ - def __init__(self, sock, timeout): + def __init__(self, sock, timeout, method): self.sock = sock + self.method = method self.raw_response = '' - self._body = None self._headers_len = 0 - self._content_len = 0 self.headers = None self.will_close = False self.status_line = '' self.status = None + self.continued = False self.http_version = None self.reason = None - self._chunked = False - self._chunked_done = False - self._chunked_until_next = 0 - self._chunked_skip_bytes = 0 - self._chunked_preloaded_block = None + self._reader = None self._read_location = 0 self._eol = EOL @@ -117,11 +112,12 @@ socket is closed, this will nearly always return False, even in cases where all the data has actually been loaded. """ - if self._chunked: - return self._chunked_done - if self._content_len == _LEN_CLOSE_IS_END: - return False - return self._body is not None and len(self._body) >= self._content_len + if self._reader: + return self._reader.done() + + def _close(self): + if self._reader is not None: + self._reader._close() def readline(self): """Read a single line from the response body. @@ -129,30 +125,34 @@ This may block until either a line ending is found or the response is complete. """ - eol = self._body.find('\n', self._read_location) - while eol == -1 and not self.complete(): + # TODO: move this into the reader interface where it can be + # smarter (and probably avoid copies) + bytes = [] + while not bytes: + try: + bytes = [self._reader.read(1)] + except _readers.ReadNotReady: + self._select() + while bytes[-1] != '\n' and not self.complete(): self._select() - eol = self._body.find('\n', self._read_location) - if eol != -1: - eol += 1 - else: - eol = len(self._body) - data = self._body[self._read_location:eol] - self._read_location = eol - return data + bytes.append(self._reader.read(1)) + if bytes[-1] != '\n': + next = self._reader.read(1) + while next and next != '\n': + bytes.append(next) + next = self._reader.read(1) + bytes.append(next) + return ''.join(bytes) def read(self, length=None): # if length is None, unbounded read while (not self.complete() # never select on a finished read and (not length # unbounded, so we wait for complete() - or (self._read_location + length) > len(self._body))): + or length > self._reader.available_data)): self._select() if not length: - length = len(self._body) - self._read_location - elif len(self._body) < (self._read_location + length): - length = len(self._body) - self._read_location - r = self._body[self._read_location:self._read_location + length] - self._read_location += len(r) + length = self._reader.available_data + r = self._reader.read(length) if self.complete() and self.will_close: self.sock.close() return r @@ -160,93 +160,35 @@ def _select(self): r, _, _ = select.select([self.sock], [], [], self._timeout) if not r: - # socket was not readable. If the response is not complete - # and we're not a _LEN_CLOSE_IS_END response, raise a timeout. - # If we are a _LEN_CLOSE_IS_END response and we have no data, - # raise a timeout. - if not (self.complete() or - (self._content_len == _LEN_CLOSE_IS_END and self._body)): + # socket was not readable. If the response is not + # complete, raise a timeout. + if not self.complete(): logger.info('timed out with timeout of %s', self._timeout) raise HTTPTimeoutException('timeout reading data') - logger.info('cl: %r body: %r', self._content_len, self._body) try: data = self.sock.recv(INCOMING_BUFFER_SIZE) - # If the socket was readable and no data was read, that - # means the socket was closed. If this isn't a - # _CLOSE_IS_END socket, then something is wrong if we're - # here (we shouldn't enter _select() if the response is - # complete), so abort. - if not data and self._content_len != _LEN_CLOSE_IS_END: - raise HTTPRemoteClosedError( - 'server appears to have closed the socket mid-response') except socket.sslerror, e: if e.args[0] != socket.SSL_ERROR_WANT_READ: raise logger.debug('SSL_WANT_READ in _select, should retry later') return True logger.debug('response read %d data during _select', len(data)) + # If the socket was readable and no data was read, that means + # the socket was closed. Inform the reader (if any) so it can + # raise an exception if this is an invalid situation. if not data: - if self.headers and self._content_len == _LEN_CLOSE_IS_END: - self._content_len = len(self._body) + if self._reader: + self._reader._close() return False else: self._load_response(data) return True - def _chunked_parsedata(self, data): - if self._chunked_preloaded_block: - data = self._chunked_preloaded_block + data - self._chunked_preloaded_block = None - while data: - logger.debug('looping with %d data remaining', len(data)) - # Slice out anything we should skip - if self._chunked_skip_bytes: - if len(data) <= self._chunked_skip_bytes: - self._chunked_skip_bytes -= len(data) - data = '' - break - else: - data = data[self._chunked_skip_bytes:] - self._chunked_skip_bytes = 0 - - # determine how much is until the next chunk - if self._chunked_until_next: - amt = self._chunked_until_next - logger.debug('reading remaining %d of existing chunk', amt) - self._chunked_until_next = 0 - body = data - else: - try: - amt, body = data.split(self._eol, 1) - except ValueError: - self._chunked_preloaded_block = data - logger.debug('saving %r as a preloaded block for chunked', - self._chunked_preloaded_block) - return - amt = int(amt, base=16) - logger.debug('reading chunk of length %d', amt) - if amt == 0: - self._chunked_done = True - - # read through end of what we have or the chunk - self._body += body[:amt] - if len(body) >= amt: - data = body[amt:] - self._chunked_skip_bytes = len(self._eol) - else: - self._chunked_until_next = amt - len(body) - self._chunked_skip_bytes = 0 - data = '' - def _load_response(self, data): - if self._chunked: - self._chunked_parsedata(data) - return - elif self._body is not None: - self._body += data - return - - # We haven't seen end of headers yet + # Being here implies we're not at the end of the headers yet, + # since at the end of this method if headers were completely + # loaded we replace this method with the load() method of the + # reader we created. self.raw_response += data # This is a bogus server with bad line endings if self._eol not in self.raw_response: @@ -270,6 +212,7 @@ http_ver, status = hdrs.split(' ', 1) if status.startswith('100'): self.raw_response = body + self.continued = True logger.debug('continue seen, setting body to %r', body) return @@ -289,23 +232,46 @@ if self._eol != EOL: hdrs = hdrs.replace(self._eol, '\r\n') headers = rfc822.Message(cStringIO.StringIO(hdrs)) + content_len = None if HDR_CONTENT_LENGTH in headers: - self._content_len = int(headers[HDR_CONTENT_LENGTH]) + content_len = int(headers[HDR_CONTENT_LENGTH]) if self.http_version == HTTP_VER_1_0: self.will_close = True elif HDR_CONNECTION_CTRL in headers: self.will_close = ( headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE) - if self._content_len == 0: - self._content_len = _LEN_CLOSE_IS_END if (HDR_XFER_ENCODING in headers and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED): - self._body = '' - self._chunked_parsedata(body) - self._chunked = True - if self._body is None: - self._body = body + self._reader = _readers.ChunkedReader(self._eol) + logger.debug('using a chunked reader') + else: + # HEAD responses are forbidden from returning a body, and + # it's implausible for a CONNECT response to use + # close-is-end logic for an OK response. + if (self.method == 'HEAD' or + (self.method == 'CONNECT' and content_len is None)): + content_len = 0 + if content_len is not None: + logger.debug('using a content-length reader with length %d', + content_len) + self._reader = _readers.ContentLengthReader(content_len) + else: + # Response body had no length specified and is not + # chunked, so the end of the body will only be + # identifiable by the termination of the socket by the + # server. My interpretation of the spec means that we + # are correct in hitting this case if + # transfer-encoding, content-length, and + # connection-control were left unspecified. + self._reader = _readers.CloseIsEndReader() + logger.debug('using a close-is-end reader') + self.will_close = True + + if body: + self._reader._load(body) + logger.debug('headers complete') self.headers = headers + self._load_response = self._reader._load class HTTPConnection(object): @@ -382,13 +348,14 @@ {}, HTTP_VER_1_0) sock.send(data) sock.setblocking(0) - r = self.response_class(sock, self.timeout) + r = self.response_class(sock, self.timeout, 'CONNECT') timeout_exc = HTTPTimeoutException( 'Timed out waiting for CONNECT response from proxy') while not r.complete(): try: if not r._select(): - raise timeout_exc + if not r.complete(): + raise timeout_exc except HTTPTimeoutException: # This raise/except pattern looks goofy, but # _select can raise the timeout as well as the @@ -527,7 +494,7 @@ out = outgoing_headers or body blocking_on_continue = False if expect_continue and not outgoing_headers and not ( - response and response.headers): + response and (response.headers or response.continued)): logger.info( 'waiting up to %s seconds for' ' continue response from server', @@ -550,11 +517,6 @@ 'server, optimistically sending request body') else: raise HTTPTimeoutException('timeout sending data') - # TODO exceptional conditions with select? (what are those be?) - # TODO if the response is loading, must we finish sending at all? - # - # Certainly not if it's going to close the connection and/or - # the response is already done...I think. was_first = first # incoming data @@ -572,11 +534,11 @@ logger.info('socket appears closed in read') self.sock = None self._current_response = None + if response is not None: + response._close() # This if/elif ladder is a bit subtle, # comments in each branch should help. - if response is not None and ( - response.complete() or - response._content_len == _LEN_CLOSE_IS_END): + if response is not None and response.complete(): # Server responded completely and then # closed the socket. We should just shut # things down and let the caller get their @@ -605,7 +567,7 @@ 'response was missing or incomplete!') logger.debug('read %d bytes in request()', len(data)) if response is None: - response = self.response_class(r[0], self.timeout) + response = self.response_class(r[0], self.timeout, method) response._load_response(data) # Jump to the next select() call so we load more # data if the server is still sending us content. @@ -613,10 +575,6 @@ except socket.error, e: if e[0] != errno.EPIPE and not was_first: raise - if (response._content_len - and response._content_len != _LEN_CLOSE_IS_END): - outgoing_headers = sent_data + outgoing_headers - reconnect('read') # outgoing data if w and out: @@ -661,7 +619,7 @@ # close if the server response said to or responded before eating # the whole request if response is None: - response = self.response_class(self.sock, self.timeout) + response = self.response_class(self.sock, self.timeout, method) complete = response.complete() data_left = bool(outgoing_headers or body) if data_left: @@ -679,7 +637,8 @@ raise httplib.ResponseNotReady() r = self._current_response while r.headers is None: - r._select() + if not r._select() and not r.complete(): + raise _readers.HTTPRemoteClosedError() if r.will_close: self.sock = None self._current_response = None @@ -705,7 +664,7 @@ class HTTPStateError(httplib.HTTPException): """Invalid internal state encountered.""" - -class HTTPRemoteClosedError(httplib.HTTPException): - """The server closed the remote socket in the middle of a response.""" +# Forward this exception type from _readers since it needs to be part +# of the public API. +HTTPRemoteClosedError = _readers.HTTPRemoteClosedError # no-check-code
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/httpclient/_readers.py Fri May 04 16:00:33 2012 -0500 @@ -0,0 +1,195 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Reader objects to abstract out different body response types. + +This module is package-private. It is not expected that these will +have any clients outside of httpplus. +""" + +import httplib +import itertools +import logging + +logger = logging.getLogger(__name__) + + +class ReadNotReady(Exception): + """Raised when read() is attempted but not enough data is loaded.""" + + +class HTTPRemoteClosedError(httplib.HTTPException): + """The server closed the remote socket in the middle of a response.""" + + +class AbstractReader(object): + """Abstract base class for response readers. + + Subclasses must implement _load, and should implement _close if + it's not an error for the server to close their socket without + some termination condition being detected during _load. + """ + def __init__(self): + self._finished = False + self._done_chunks = [] + + @property + def available_data(self): + return sum(map(len, self._done_chunks)) + + def done(self): + return self._finished + + def read(self, amt): + if self.available_data < amt and not self._finished: + raise ReadNotReady() + need = [amt] + def pred(s): + needed = need[0] > 0 + need[0] -= len(s) + return needed + blocks = list(itertools.takewhile(pred, self._done_chunks)) + self._done_chunks = self._done_chunks[len(blocks):] + over_read = sum(map(len, blocks)) - amt + if over_read > 0 and blocks: + logger.debug('need to reinsert %d data into done chunks', over_read) + last = blocks[-1] + blocks[-1], reinsert = last[:-over_read], last[-over_read:] + self._done_chunks.insert(0, reinsert) + result = ''.join(blocks) + assert len(result) == amt or (self._finished and len(result) < amt) + return result + + def _load(self, data): # pragma: no cover + """Subclasses must implement this. + + As data is available to be read out of this object, it should + be placed into the _done_chunks list. Subclasses should not + rely on data remaining in _done_chunks forever, as it may be + reaped if the client is parsing data as it comes in. + """ + raise NotImplementedError + + def _close(self): + """Default implementation of close. + + The default implementation assumes that the reader will mark + the response as finished on the _finished attribute once the + entire response body has been read. In the event that this is + not true, the subclass should override the implementation of + close (for example, close-is-end responses have to set + self._finished in the close handler.) + """ + if not self._finished: + raise HTTPRemoteClosedError( + 'server appears to have closed the socket mid-response') + + +class AbstractSimpleReader(AbstractReader): + """Abstract base class for simple readers that require no response decoding. + + Examples of such responses are Connection: Close (close-is-end) + and responses that specify a content length. + """ + def _load(self, data): + if data: + assert not self._finished, ( + 'tried to add data (%r) to a closed reader!' % data) + logger.debug('%s read an addtional %d data', self.name, len(data)) + self._done_chunks.append(data) + + +class CloseIsEndReader(AbstractSimpleReader): + """Reader for responses that specify Connection: Close for length.""" + name = 'close-is-end' + + def _close(self): + logger.info('Marking close-is-end reader as closed.') + self._finished = True + + +class ContentLengthReader(AbstractSimpleReader): + """Reader for responses that specify an exact content length.""" + name = 'content-length' + + def __init__(self, amount): + AbstractReader.__init__(self) + self._amount = amount + if amount == 0: + self._finished = True + self._amount_seen = 0 + + def _load(self, data): + AbstractSimpleReader._load(self, data) + self._amount_seen += len(data) + if self._amount_seen >= self._amount: + self._finished = True + logger.debug('content-length read complete') + + +class ChunkedReader(AbstractReader): + """Reader for chunked transfer encoding responses.""" + def __init__(self, eol): + AbstractReader.__init__(self) + self._eol = eol + self._leftover_skip_amt = 0 + self._leftover_data = '' + + def _load(self, data): + assert not self._finished, 'tried to add data to a closed reader!' + logger.debug('chunked read an addtional %d data', len(data)) + position = 0 + if self._leftover_data: + logger.debug('chunked reader trying to finish block from leftover data') + # TODO: avoid this string concatenation if possible + data = self._leftover_data + data + position = self._leftover_skip_amt + self._leftover_data = '' + self._leftover_skip_amt = 0 + datalen = len(data) + while position < datalen: + split = data.find(self._eol, position) + if split == -1: + self._leftover_data = data + self._leftover_skip_amt = position + return + amt = int(data[position:split], base=16) + block_start = split + len(self._eol) + # If the whole data chunk plus the eol trailer hasn't + # loaded, we'll wait for the next load. + if block_start + amt + len(self._eol) > len(data): + self._leftover_data = data + self._leftover_skip_amt = position + return + if amt == 0: + self._finished = True + logger.debug('closing chunked redaer due to chunk of length 0') + return + self._done_chunks.append(data[block_start:block_start + amt]) + position = block_start + amt + len(self._eol) +# no-check-code
--- a/mercurial/httpclient/tests/simple_http_test.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/tests/simple_http_test.py Fri May 04 16:00:33 2012 -0500 @@ -29,7 +29,7 @@ import socket import unittest -import http +import httpplus # relative import to ease embedding the library import util @@ -38,7 +38,7 @@ class SimpleHttpTest(util.HttpTestBase, unittest.TestCase): def _run_simple_test(self, host, server_data, expected_req, expected_data): - con = http.HTTPConnection(host) + con = httpplus.HTTPConnection(host) con._connect() con.sock.data = server_data con.request('GET', '/') @@ -47,9 +47,9 @@ self.assertEqual(expected_data, con.getresponse().read()) def test_broken_data_obj(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() - self.assertRaises(http.BadRequestData, + self.assertRaises(httpplus.BadRequestData, con.request, 'POST', '/', body=1) def test_no_keepalive_http_1_0(self): @@ -74,7 +74,7 @@ fncache dotencode """ - con = http.HTTPConnection('localhost:9999') + con = httpplus.HTTPConnection('localhost:9999') con._connect() con.sock.data = [expected_response_headers, expected_response_body] con.request('GET', '/remote/.hg/requires', @@ -95,7 +95,7 @@ self.assert_(resp.sock.closed) def test_multiline_header(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() con.sock.data = ['HTTP/1.1 200 OK\r\n', 'Server: BogusServer 1.0\r\n', @@ -122,7 +122,7 @@ self.assertEqual(con.sock.closed, False) def testSimpleRequest(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() con.sock.data = ['HTTP/1.1 200 OK\r\n', 'Server: BogusServer 1.0\r\n', @@ -149,12 +149,13 @@ resp.headers.getheaders('server')) def testHeaderlessResponse(self): - con = http.HTTPConnection('1.2.3.4', use_ssl=False) + con = httpplus.HTTPConnection('1.2.3.4', use_ssl=False) con._connect() con.sock.data = ['HTTP/1.1 200 OK\r\n', '\r\n' '1234567890' ] + con.sock.close_on_empty = True con.request('GET', '/') expected_req = ('GET / HTTP/1.1\r\n' @@ -169,7 +170,30 @@ self.assertEqual(resp.status, 200) def testReadline(self): - con = http.HTTPConnection('1.2.3.4') + con = httpplus.HTTPConnection('1.2.3.4') + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Connection: Close\r\n', + '\r\n' + '1\n2\nabcdefg\n4\n5'] + con.sock.close_on_empty = True + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + con.request('GET', '/') + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + r = con.getresponse() + for expected in ['1\n', '2\n', 'abcdefg\n', '4\n', '5']: + actual = r.readline() + self.assertEqual(expected, actual, + 'Expected %r, got %r' % (expected, actual)) + + def testReadlineTrickle(self): + con = httpplus.HTTPConnection('1.2.3.4') con._connect() # make sure it trickles in one byte at a time # so that we touch all the cases in readline @@ -179,6 +203,7 @@ 'Connection: Close\r\n', '\r\n' '1\n2\nabcdefg\n4\n5'])) + con.sock.close_on_empty = True expected_req = ('GET / HTTP/1.1\r\n' 'Host: 1.2.3.4\r\n' @@ -193,6 +218,59 @@ self.assertEqual(expected, actual, 'Expected %r, got %r' % (expected, actual)) + def testVariousReads(self): + con = httpplus.HTTPConnection('1.2.3.4') + con._connect() + # make sure it trickles in one byte at a time + # so that we touch all the cases in readline + con.sock.data = list(''.join( + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Connection: Close\r\n', + '\r\n' + '1\n2', + '\na', 'bc', + 'defg\n4\n5'])) + con.sock.close_on_empty = True + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + con.request('GET', '/') + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + r = con.getresponse() + for read_amt, expect in [(1, '1'), (1, '\n'), + (4, '2\nab'), + ('line', 'cdefg\n'), + (None, '4\n5')]: + if read_amt == 'line': + self.assertEqual(expect, r.readline()) + else: + self.assertEqual(expect, r.read(read_amt)) + + def testZeroLengthBody(self): + con = httpplus.HTTPConnection('1.2.3.4') + con._connect() + # make sure it trickles in one byte at a time + # so that we touch all the cases in readline + con.sock.data = list(''.join( + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-length: 0\r\n', + '\r\n'])) + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + con.request('GET', '/') + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + r = con.getresponse() + self.assertEqual('', r.read()) + def testIPv6(self): self._run_simple_test('[::1]:8221', ['HTTP/1.1 200 OK\r\n', @@ -226,7 +304,7 @@ '1234567890') def testEarlyContinueResponse(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.data = ['HTTP/1.1 403 Forbidden\r\n', @@ -240,8 +318,23 @@ self.assertEqual("You can't do that.", con.getresponse().read()) self.assertEqual(sock.closed, True) + def testEarlyContinueResponseNoContentLength(self): + con = httpplus.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 403 Forbidden\r\n', + 'Server: BogusServer 1.0\r\n', + '\r\n' + "You can't do that."] + sock.close_on_empty = True + expected_req = self.doPost(con, expect_body=False) + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assertStringEqual(expected_req, sock.sent) + self.assertEqual("You can't do that.", con.getresponse().read()) + self.assertEqual(sock.closed, True) + def testDeniedAfterContinueTimeoutExpires(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.data = ['HTTP/1.1 403 Forbidden\r\n', @@ -269,7 +362,7 @@ self.assertEqual(sock.closed, True) def testPostData(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.read_wait_sentinel = 'POST data' @@ -286,7 +379,7 @@ self.assertEqual(sock.closed, False) def testServerWithoutContinue(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.read_wait_sentinel = 'POST data' @@ -302,7 +395,7 @@ self.assertEqual(sock.closed, False) def testServerWithSlowContinue(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.read_wait_sentinel = 'POST data' @@ -321,7 +414,7 @@ self.assertEqual(sock.closed, False) def testSlowConnection(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() # simulate one byte arriving at a time, to check for various # corner cases @@ -340,12 +433,26 @@ self.assertEqual(expected_req, con.sock.sent) self.assertEqual('1234567890', con.getresponse().read()) + def testCloseAfterNotAllOfHeaders(self): + con = httpplus.HTTPConnection('1.2.3.4:80') + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: NO CARRIER'] + con.sock.close_on_empty = True + con.request('GET', '/') + self.assertRaises(httpplus.HTTPRemoteClosedError, + con.getresponse) + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + def testTimeout(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() con.sock.data = [] con.request('GET', '/') - self.assertRaises(http.HTTPTimeoutException, + self.assertRaises(httpplus.HTTPTimeoutException, con.getresponse) expected_req = ('GET / HTTP/1.1\r\n' @@ -370,7 +477,7 @@ return s socket.socket = closingsocket - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() con.request('GET', '/') r1 = con.getresponse() @@ -381,7 +488,7 @@ self.assertEqual(2, len(sockets)) def test_server_closes_before_end_of_body(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() s = con.sock s.data = ['HTTP/1.1 200 OK\r\n', @@ -393,9 +500,9 @@ s.close_on_empty = True con.request('GET', '/') r1 = con.getresponse() - self.assertRaises(http.HTTPRemoteClosedError, r1.read) + self.assertRaises(httpplus.HTTPRemoteClosedError, r1.read) def test_no_response_raises_response_not_ready(self): - con = http.HTTPConnection('foo') - self.assertRaises(http.httplib.ResponseNotReady, con.getresponse) + con = httpplus.HTTPConnection('foo') + self.assertRaises(httpplus.httplib.ResponseNotReady, con.getresponse) # no-check-code
--- a/mercurial/httpclient/tests/test_bogus_responses.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/tests/test_bogus_responses.py Fri May 04 16:00:33 2012 -0500 @@ -34,7 +34,7 @@ """ import unittest -import http +import httpplus # relative import to ease embedding the library import util @@ -43,7 +43,7 @@ class SimpleHttpTest(util.HttpTestBase, unittest.TestCase): def bogusEOL(self, eol): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() con.sock.data = ['HTTP/1.1 200 OK%s' % eol, 'Server: BogusServer 1.0%s' % eol,
--- a/mercurial/httpclient/tests/test_chunked_transfer.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/tests/test_chunked_transfer.py Fri May 04 16:00:33 2012 -0500 @@ -29,7 +29,7 @@ import cStringIO import unittest -import http +import httpplus # relative import to ease embedding the library import util @@ -50,7 +50,7 @@ class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase): def testChunkedUpload(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.read_wait_sentinel = '0\r\n\r\n' @@ -77,7 +77,7 @@ self.assertEqual(sock.closed, False) def testChunkedDownload(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.data = ['HTTP/1.1 200 OK\r\n', @@ -85,14 +85,31 @@ 'transfer-encoding: chunked', '\r\n\r\n', chunkedblock('hi '), - chunkedblock('there'), + ] + list(chunkedblock('there')) + [ chunkedblock(''), ] con.request('GET', '/') self.assertStringEqual('hi there', con.getresponse().read()) + def testChunkedDownloadOddReadBoundaries(self): + con = httpplus.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'transfer-encoding: chunked', + '\r\n\r\n', + chunkedblock('hi '), + ] + list(chunkedblock('there')) + [ + chunkedblock(''), + ] + con.request('GET', '/') + resp = con.getresponse() + for amt, expect in [(1, 'h'), (5, 'i the'), (100, 're')]: + self.assertEqual(expect, resp.read(amt)) + def testChunkedDownloadBadEOL(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.data = ['HTTP/1.1 200 OK\n', @@ -107,7 +124,7 @@ self.assertStringEqual('hi there', con.getresponse().read()) def testChunkedDownloadPartialChunkBadEOL(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.data = ['HTTP/1.1 200 OK\n', @@ -122,7 +139,7 @@ con.getresponse().read()) def testChunkedDownloadPartialChunk(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock sock.data = ['HTTP/1.1 200 OK\r\n', @@ -136,7 +153,7 @@ con.getresponse().read()) def testChunkedDownloadEarlyHangup(self): - con = http.HTTPConnection('1.2.3.4:80') + con = httpplus.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock broken = chunkedblock('hi'*20)[:-1] @@ -149,5 +166,5 @@ sock.close_on_empty = True con.request('GET', '/') resp = con.getresponse() - self.assertRaises(http.HTTPRemoteClosedError, resp.read) + self.assertRaises(httpplus.HTTPRemoteClosedError, resp.read) # no-check-code
--- a/mercurial/httpclient/tests/test_proxy_support.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/tests/test_proxy_support.py Fri May 04 16:00:33 2012 -0500 @@ -29,13 +29,13 @@ import unittest import socket -import http +import httpplus # relative import to ease embedding the library import util -def make_preloaded_socket(data): +def make_preloaded_socket(data, close=False): """Make a socket pre-loaded with data so it can be read during connect. Useful for https proxy tests because we have to read from the @@ -44,6 +44,7 @@ def s(*args, **kwargs): sock = util.MockSocket(*args, **kwargs) sock.early_data = data[:] + sock.close_on_empty = close return sock return s @@ -51,7 +52,7 @@ class ProxyHttpTest(util.HttpTestBase, unittest.TestCase): def _run_simple_test(self, host, server_data, expected_req, expected_data): - con = http.HTTPConnection(host) + con = httpplus.HTTPConnection(host) con._connect() con.sock.data = server_data con.request('GET', '/') @@ -60,7 +61,7 @@ self.assertEqual(expected_data, con.getresponse().read()) def testSimpleRequest(self): - con = http.HTTPConnection('1.2.3.4:80', + con = httpplus.HTTPConnection('1.2.3.4:80', proxy_hostport=('magicproxy', 4242)) con._connect() con.sock.data = ['HTTP/1.1 200 OK\r\n', @@ -88,7 +89,7 @@ resp.headers.getheaders('server')) def testSSLRequest(self): - con = http.HTTPConnection('1.2.3.4:443', + con = httpplus.HTTPConnection('1.2.3.4:443', proxy_hostport=('magicproxy', 4242)) socket.socket = make_preloaded_socket( ['HTTP/1.1 200 OK\r\n', @@ -124,12 +125,47 @@ self.assertEqual(['BogusServer 1.0'], resp.headers.getheaders('server')) - def testSSLProxyFailure(self): - con = http.HTTPConnection('1.2.3.4:443', + def testSSLRequestNoConnectBody(self): + con = httpplus.HTTPConnection('1.2.3.4:443', proxy_hostport=('magicproxy', 4242)) socket.socket = make_preloaded_socket( - ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n']) - self.assertRaises(http.HTTPProxyConnectFailedException, con._connect) - self.assertRaises(http.HTTPProxyConnectFailedException, + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + '\r\n']) + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + connect_sent = con.sock.sent + con.sock.sent = '' + con.request('GET', '/') + + expected_connect = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n' + '\r\n') + expected_request = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('127.0.0.42', 4242), con.sock.sa) + self.assertStringEqual(expected_connect, connect_sent) + self.assertStringEqual(expected_request, con.sock.sent) + resp = con.getresponse() + self.assertEqual(resp.status, 200) + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['BogusServer 1.0'], + resp.headers.getheaders('server')) + + def testSSLProxyFailure(self): + con = httpplus.HTTPConnection('1.2.3.4:443', + proxy_hostport=('magicproxy', 4242)) + socket.socket = make_preloaded_socket( + ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n'], close=True) + self.assertRaises(httpplus.HTTPProxyConnectFailedException, con._connect) + self.assertRaises(httpplus.HTTPProxyConnectFailedException, con.request, 'GET', '/') # no-check-code
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/httpclient/tests/test_readers.py Fri May 04 16:00:33 2012 -0500 @@ -0,0 +1,70 @@ +# Copyright 2010, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from httpplus import _readers + +def chunkedblock(x, eol='\r\n'): + r"""Make a chunked transfer-encoding block. + + >>> chunkedblock('hi') + '2\r\nhi\r\n' + >>> chunkedblock('hi' * 10) + '14\r\nhihihihihihihihihihi\r\n' + >>> chunkedblock('hi', eol='\n') + '2\nhi\n' + """ + return ''.join((hex(len(x))[2:], eol, x, eol)) + +corpus = 'foo\r\nbar\r\nbaz\r\n' + + +class ChunkedReaderTest(unittest.TestCase): + def test_many_block_boundaries(self): + for step in xrange(1, len(corpus)): + data = ''.join(chunkedblock(corpus[start:start+step]) for + start in xrange(0, len(corpus), step)) + for istep in xrange(1, len(data)): + rdr = _readers.ChunkedReader('\r\n') + print 'step', step, 'load', istep + for start in xrange(0, len(data), istep): + rdr._load(data[start:start+istep]) + rdr._load(chunkedblock('')) + self.assertEqual(corpus, rdr.read(len(corpus) + 1)) + + def test_small_chunk_blocks_large_wire_blocks(self): + data = ''.join(map(chunkedblock, corpus)) + chunkedblock('') + rdr = _readers.ChunkedReader('\r\n') + for start in xrange(0, len(data), 4): + d = data[start:start + 4] + if d: + rdr._load(d) + self.assertEqual(corpus, rdr.read(len(corpus)+100)) +# no-check-code
--- a/mercurial/httpclient/tests/test_ssl.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/tests/test_ssl.py Fri May 04 16:00:33 2012 -0500 @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -import http +import httpplus # relative import to ease embedding the library import util @@ -37,7 +37,7 @@ class HttpSslTest(util.HttpTestBase, unittest.TestCase): def testSslRereadRequired(self): - con = http.HTTPConnection('1.2.3.4:443') + con = httpplus.HTTPConnection('1.2.3.4:443') con._connect() # extend the list instead of assign because of how # MockSSLSocket works. @@ -66,7 +66,7 @@ resp.headers.getheaders('server')) def testSslRereadInEarlyResponse(self): - con = http.HTTPConnection('1.2.3.4:443') + con = httpplus.HTTPConnection('1.2.3.4:443') con._connect() con.sock.early_data = ['HTTP/1.1 200 OK\r\n', 'Server: BogusServer 1.0\r\n',
--- a/mercurial/httpclient/tests/util.py Sat May 12 00:06:11 2012 +0200 +++ b/mercurial/httpclient/tests/util.py Fri May 04 16:00:33 2012 -0500 @@ -29,7 +29,7 @@ import difflib import socket -import http +import httpplus class MockSocket(object): @@ -57,7 +57,7 @@ self.remote_closed = self.closed = False self.close_on_empty = False self.sent = '' - self.read_wait_sentinel = http._END_HEADERS + self.read_wait_sentinel = httpplus._END_HEADERS def close(self): self.closed = True @@ -86,7 +86,7 @@ @property def ready_for_read(self): - return ((self.early_data and http._END_HEADERS in self.sent) + return ((self.early_data and httpplus._END_HEADERS in self.sent) or (self.read_wait_sentinel in self.sent and self.data) or self.closed or self.remote_closed) @@ -132,7 +132,7 @@ def mocksslwrap(sock, keyfile=None, certfile=None, - server_side=False, cert_reqs=http.socketutil.CERT_NONE, + server_side=False, cert_reqs=httpplus.socketutil.CERT_NONE, ssl_version=None, ca_certs=None, do_handshake_on_connect=True, suppress_ragged_eofs=True): @@ -156,16 +156,16 @@ self.orig_getaddrinfo = socket.getaddrinfo socket.getaddrinfo = mockgetaddrinfo - self.orig_select = http.select.select - http.select.select = mockselect + self.orig_select = httpplus.select.select + httpplus.select.select = mockselect - self.orig_sslwrap = http.socketutil.wrap_socket - http.socketutil.wrap_socket = mocksslwrap + self.orig_sslwrap = httpplus.socketutil.wrap_socket + httpplus.socketutil.wrap_socket = mocksslwrap def tearDown(self): socket.socket = self.orig_socket - http.select.select = self.orig_select - http.socketutil.wrap_socket = self.orig_sslwrap + httpplus.select.select = self.orig_select + httpplus.socketutil.wrap_socket = self.orig_sslwrap socket.getaddrinfo = self.orig_getaddrinfo def assertStringEqual(self, l, r):