Mercurial > hg
changeset 14376:a75e0f4ba0ab
httpclient: import revision fc731618702a of py-nonblocking-http
| author | Augie Fackler <durin42@gmail.com> |
|---|---|
| date | Tue, 17 May 2011 10:28:03 -0500 |
| parents | 436e5379d7ba |
| children | 1dab7afff11f ce6f9a234d7e |
| files | mercurial/httpclient/__init__.py mercurial/httpclient/tests/simple_http_test.py mercurial/httpclient/tests/test_chunked_transfer.py mercurial/httpclient/tests/test_proxy_support.py mercurial/httpclient/tests/test_ssl.py mercurial/httpclient/tests/util.py |
| diffstat | 6 files changed, 120 insertions(+), 50 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/httpclient/__init__.py Mon May 16 16:59:45 2011 -0500 +++ b/mercurial/httpclient/__init__.py Tue May 17 10:28:03 2011 -0500 @@ -112,6 +112,10 @@ def complete(self): """Returns true if this response is completely loaded. + + Note that if this is a connection where complete means the + socket is closed, this will nearly always return False, even + in cases where all the data has actually been loaded. """ if self._chunked: return self._chunked_done @@ -174,10 +178,7 @@ return True logger.debug('response read %d data during _select', len(data)) if not data: - if not self.headers: - self._load_response(self._end_headers) - self._content_len = 0 - elif self._content_len == _LEN_CLOSE_IS_END: + if self.headers and self._content_len == _LEN_CLOSE_IS_END: self._content_len = len(self._body) return False else: @@ -561,17 +562,46 @@ continue if not data: logger.info('socket appears closed in read') - outgoing_headers = body = None - break + self.sock = None + self._current_response = None + # This if/elif ladder is a bit subtle, + # comments in each branch should help. + if response is not None and ( + response.complete() or + response._content_len == _LEN_CLOSE_IS_END): + # Server responded completely and then + # closed the socket. We should just shut + # things down and let the caller get their + # response. + logger.info('Got an early response, ' + 'aborting remaining request.') + break + elif was_first and response is None: + # Most likely a keepalive that got killed + # on the server's end. Commonly happens + # after getting a really large response + # from the server. + logger.info( + 'Connection appeared closed in read on first' + ' request loop iteration, will retry.') + reconnect('read') + continue + else: + # We didn't just send the first data hunk, + # and either have a partial response or no + # response at all. There's really nothing + # meaningful we can do here. + raise HTTPStateError( + 'Connection appears closed after ' + 'some request data was written, but the ' + 'response was missing or incomplete!') + logger.debug('read %d bytes in request()', len(data)) if response is None: response = self.response_class(r[0], self.timeout) response._load_response(data) - if (response._content_len == _LEN_CLOSE_IS_END - and len(data) == 0): - response._content_len = len(response._body) - if response.complete(): - w = [] - response.will_close = True + # Jump to the next select() call so we load more + # data if the server is still sending us content. + continue except socket.error, e: if e[0] != errno.EPIPE and not was_first: raise @@ -662,4 +692,7 @@ class HTTPProxyConnectFailedException(httplib.HTTPException): """Connecting to the HTTP proxy failed.""" + +class HTTPStateError(httplib.HTTPException): + """Invalid internal state encountered.""" # no-check-code
--- a/mercurial/httpclient/tests/simple_http_test.py Mon May 16 16:59:45 2011 -0500 +++ b/mercurial/httpclient/tests/simple_http_test.py Tue May 17 10:28:03 2011 -0500 @@ -26,6 +26,7 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import socket import unittest import http @@ -39,7 +40,7 @@ def _run_simple_test(self, host, server_data, expected_req, expected_data): con = http.HTTPConnection(host) con._connect() - con.sock.data.extend(server_data) + con.sock.data = server_data con.request('GET', '/') self.assertStringEqual(expected_req, con.sock.sent) @@ -353,4 +354,33 @@ self.assertEqual(('1.2.3.4', 80), con.sock.sa) self.assertEqual(expected_req, con.sock.sent) + + def test_conn_keep_alive_but_server_close_anyway(self): + sockets = [] + def closingsocket(*args, **kwargs): + s = util.MockSocket(*args, **kwargs) + sockets.append(s) + s.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Connection: Keep-Alive\r\n', + 'Content-Length: 16', + '\r\n\r\n', + 'You can do that.'] + s.close_on_empty = True + return s + + socket.socket = closingsocket + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + con.request('GET', '/') + r1 = con.getresponse() + r1.read() + self.assertFalse(con.sock.closed) + self.assert_(con.sock.remote_closed) + con.request('GET', '/') + self.assertEqual(2, len(sockets)) + + def test_no_response_raises_response_not_ready(self): + con = http.HTTPConnection('foo') + self.assertRaises(http.httplib.ResponseNotReady, con.getresponse) # no-check-code
--- a/mercurial/httpclient/tests/test_chunked_transfer.py Mon May 16 16:59:45 2011 -0500 +++ b/mercurial/httpclient/tests/test_chunked_transfer.py Tue May 17 10:28:03 2011 -0500 @@ -53,7 +53,7 @@ con = http.HTTPConnection('1.2.3.4:80') con._connect() sock = con.sock - sock.read_wait_sentinel = 'end-of-body' + sock.read_wait_sentinel = '0\r\n\r\n' sock.data = ['HTTP/1.1 200 OK\r\n', 'Server: BogusServer 1.0\r\n', 'Content-Length: 6',
--- a/mercurial/httpclient/tests/test_proxy_support.py Mon May 16 16:59:45 2011 -0500 +++ b/mercurial/httpclient/tests/test_proxy_support.py Tue May 17 10:28:03 2011 -0500 @@ -43,7 +43,7 @@ """ def s(*args, **kwargs): sock = util.MockSocket(*args, **kwargs) - sock.data = data[:] + sock.early_data = data[:] return sock return s @@ -97,24 +97,27 @@ '\r\n' '1234567890']) con._connect() - con.sock.data.extend(['HTTP/1.1 200 OK\r\n', - 'Server: BogusServer 1.0\r\n', - 'Content-Length: 10\r\n', - '\r\n' - '1234567890' - ]) + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + connect_sent = con.sock.sent + con.sock.sent = '' con.request('GET', '/') - expected_req = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n' - 'Host: 1.2.3.4\r\n' - 'accept-encoding: identity\r\n' - '\r\n' - 'GET / HTTP/1.1\r\n' - 'Host: 1.2.3.4\r\n' - 'accept-encoding: identity\r\n\r\n') + expected_connect = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n' + '\r\n') + expected_request = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') self.assertEqual(('127.0.0.42', 4242), con.sock.sa) - self.assertStringEqual(expected_req, con.sock.sent) + self.assertStringEqual(expected_connect, connect_sent) + self.assertStringEqual(expected_request, con.sock.sent) resp = con.getresponse() self.assertEqual(resp.status, 200) self.assertEqual('1234567890', resp.read())
--- a/mercurial/httpclient/tests/test_ssl.py Mon May 16 16:59:45 2011 -0500 +++ b/mercurial/httpclient/tests/test_ssl.py Tue May 17 10:28:03 2011 -0500 @@ -41,15 +41,15 @@ con._connect() # extend the list instead of assign because of how # MockSSLSocket works. - con.sock.data.extend(['HTTP/1.1 200 OK\r\n', - 'Server: BogusServer 1.0\r\n', - 'MultiHeader: Value\r\n' - 'MultiHeader: Other Value\r\n' - 'MultiHeader: One More!\r\n' - 'Content-Length: 10\r\n', - '\r\n' - '1234567890' - ]) + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'MultiHeader: Value\r\n' + 'MultiHeader: Other Value\r\n' + 'MultiHeader: One More!\r\n' + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] con.request('GET', '/') expected_req = ('GET / HTTP/1.1\r\n' @@ -68,17 +68,15 @@ def testSslRereadInEarlyResponse(self): con = http.HTTPConnection('1.2.3.4:443') con._connect() - # extend the list instead of assign because of how - # MockSSLSocket works. - con.sock.early_data.extend(['HTTP/1.1 200 OK\r\n', - 'Server: BogusServer 1.0\r\n', - 'MultiHeader: Value\r\n' - 'MultiHeader: Other Value\r\n' - 'MultiHeader: One More!\r\n' - 'Content-Length: 10\r\n', - '\r\n' - '1234567890' - ]) + con.sock.early_data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'MultiHeader: Value\r\n' + 'MultiHeader: Other Value\r\n' + 'MultiHeader: One More!\r\n' + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] expected_req = self.doPost(con, False) self.assertEqual(None, con.sock, @@ -92,3 +90,4 @@ resp.headers.getheaders('multiheader')) self.assertEqual(['BogusServer 1.0'], resp.headers.getheaders('server')) +# no-check-code
--- a/mercurial/httpclient/tests/util.py Mon May 16 16:59:45 2011 -0500 +++ b/mercurial/httpclient/tests/util.py Tue May 17 10:28:03 2011 -0500 @@ -88,7 +88,7 @@ def ready_for_read(self): return ((self.early_data and http._END_HEADERS in self.sent) or (self.read_wait_sentinel in self.sent and self.data) - or self.closed) + or self.closed or self.remote_closed) def send(self, data): # this is a horrible mock, but nothing needs us to raise the @@ -117,6 +117,11 @@ def __getattr__(self, key): return getattr(self._sock, key) + def __setattr__(self, key, value): + if key not in ('_sock', '_fail_recv'): + return setattr(self._sock, key, value) + return object.__setattr__(self, key, value) + def recv(self, amt=-1): try: if self._fail_recv: