Mercurial > hg
comparison mercurial/httpclient/__init__.py @ 19182:fae47ecaa952
httpclient: upgrade to fe8c09e4db64 of httpplus
author | Augie Fackler <raf@durin42.com> |
---|---|
date | Sat, 11 May 2013 20:25:15 -0500 |
parents | 36733ab7fa05 |
children | 42fcb2f7787d |
comparison
equal
deleted
inserted
replaced
19181:8c2fdf7d5645 | 19182:fae47ecaa952 |
---|---|
35 * uses select() to block for incoming data | 35 * uses select() to block for incoming data |
36 * notices when the server responds early to a request | 36 * notices when the server responds early to a request |
37 * implements ssl inline instead of in a different class | 37 * implements ssl inline instead of in a different class |
38 """ | 38 """ |
39 | 39 |
40 # Many functions in this file have too many arguments. | |
41 # pylint: disable=R0913 | |
42 | |
40 import cStringIO | 43 import cStringIO |
41 import errno | 44 import errno |
42 import httplib | 45 import httplib |
43 import logging | 46 import logging |
44 import rfc822 | 47 import rfc822 |
115 if self._reader: | 118 if self._reader: |
116 return self._reader.done() | 119 return self._reader.done() |
117 | 120 |
118 def _close(self): | 121 def _close(self): |
119 if self._reader is not None: | 122 if self._reader is not None: |
123 # We're a friend of the reader class here. | |
124 # pylint: disable=W0212 | |
120 self._reader._close() | 125 self._reader._close() |
121 | 126 |
122 def readline(self): | 127 def readline(self): |
123 """Read a single line from the response body. | 128 """Read a single line from the response body. |
124 | 129 |
135 self._select() | 140 self._select() |
136 | 141 |
137 return ''.join(blocks) | 142 return ''.join(blocks) |
138 | 143 |
139 def read(self, length=None): | 144 def read(self, length=None): |
145 """Read data from the response body.""" | |
140 # if length is None, unbounded read | 146 # if length is None, unbounded read |
141 while (not self.complete() # never select on a finished read | 147 while (not self.complete() # never select on a finished read |
142 and (not length # unbounded, so we wait for complete() | 148 and (not length # unbounded, so we wait for complete() |
143 or length > self._reader.available_data)): | 149 or length > self._reader.available_data)): |
144 self._select() | 150 self._select() |
148 if self.complete() and self.will_close: | 154 if self.complete() and self.will_close: |
149 self.sock.close() | 155 self.sock.close() |
150 return r | 156 return r |
151 | 157 |
152 def _select(self): | 158 def _select(self): |
153 r, _, _ = select.select([self.sock], [], [], self._timeout) | 159 r, unused_write, unused_err = select.select( |
160 [self.sock], [], [], self._timeout) | |
154 if not r: | 161 if not r: |
155 # socket was not readable. If the response is not | 162 # socket was not readable. If the response is not |
156 # complete, raise a timeout. | 163 # complete, raise a timeout. |
157 if not self.complete(): | 164 if not self.complete(): |
158 logger.info('timed out with timeout of %s', self._timeout) | 165 logger.info('timed out with timeout of %s', self._timeout) |
168 # If the socket was readable and no data was read, that means | 175 # If the socket was readable and no data was read, that means |
169 # the socket was closed. Inform the reader (if any) so it can | 176 # the socket was closed. Inform the reader (if any) so it can |
170 # raise an exception if this is an invalid situation. | 177 # raise an exception if this is an invalid situation. |
171 if not data: | 178 if not data: |
172 if self._reader: | 179 if self._reader: |
180 # We're a friend of the reader class here. | |
181 # pylint: disable=W0212 | |
173 self._reader._close() | 182 self._reader._close() |
174 return False | 183 return False |
175 else: | 184 else: |
176 self._load_response(data) | 185 self._load_response(data) |
177 return True | 186 return True |
178 | 187 |
179 def _load_response(self, data): | 188 # This method gets replaced by _load later, which confuses pylint. |
189 def _load_response(self, data): # pylint: disable=E0202 | |
180 # Being here implies we're not at the end of the headers yet, | 190 # Being here implies we're not at the end of the headers yet, |
181 # since at the end of this method if headers were completely | 191 # since at the end of this method if headers were completely |
182 # loaded we replace this method with the load() method of the | 192 # loaded we replace this method with the load() method of the |
183 # reader we created. | 193 # reader we created. |
184 self.raw_response += data | 194 self.raw_response += data |
199 if self._end_headers not in self.raw_response or self.headers: | 209 if self._end_headers not in self.raw_response or self.headers: |
200 return | 210 return |
201 | 211 |
202 # handle 100-continue response | 212 # handle 100-continue response |
203 hdrs, body = self.raw_response.split(self._end_headers, 1) | 213 hdrs, body = self.raw_response.split(self._end_headers, 1) |
204 http_ver, status = hdrs.split(' ', 1) | 214 unused_http_ver, status = hdrs.split(' ', 1) |
205 if status.startswith('100'): | 215 if status.startswith('100'): |
206 self.raw_response = body | 216 self.raw_response = body |
207 self.continued = True | 217 self.continued = True |
208 logger.debug('continue seen, setting body to %r', body) | 218 logger.debug('continue seen, setting body to %r', body) |
209 return | 219 return |
258 self._reader = _readers.CloseIsEndReader() | 268 self._reader = _readers.CloseIsEndReader() |
259 logger.debug('using a close-is-end reader') | 269 logger.debug('using a close-is-end reader') |
260 self.will_close = True | 270 self.will_close = True |
261 | 271 |
262 if body: | 272 if body: |
273 # We're a friend of the reader class here. | |
274 # pylint: disable=W0212 | |
263 self._reader._load(body) | 275 self._reader._load(body) |
264 logger.debug('headers complete') | 276 logger.debug('headers complete') |
265 self.headers = headers | 277 self.headers = headers |
278 # We're a friend of the reader class here. | |
279 # pylint: disable=W0212 | |
266 self._load_response = self._reader._load | 280 self._load_response = self._reader._load |
267 | 281 |
268 | 282 |
269 class HTTPConnection(object): | 283 class HTTPConnection(object): |
270 """Connection to a single http server. | 284 """Connection to a single http server. |
333 self._proxy_host, self._proxy_port) | 347 self._proxy_host, self._proxy_port) |
334 sock = socketutil.create_connection((self._proxy_host, | 348 sock = socketutil.create_connection((self._proxy_host, |
335 self._proxy_port)) | 349 self._proxy_port)) |
336 if self.ssl: | 350 if self.ssl: |
337 # TODO proxy header support | 351 # TODO proxy header support |
338 data = self.buildheaders('CONNECT', '%s:%d' % (self.host, | 352 data = self._buildheaders('CONNECT', '%s:%d' % (self.host, |
339 self.port), | 353 self.port), |
340 {}, HTTP_VER_1_0) | 354 {}, HTTP_VER_1_0) |
341 sock.send(data) | 355 sock.send(data) |
342 sock.setblocking(0) | 356 sock.setblocking(0) |
343 r = self.response_class(sock, self.timeout, 'CONNECT') | 357 r = self.response_class(sock, self.timeout, 'CONNECT') |
344 timeout_exc = HTTPTimeoutException( | 358 timeout_exc = HTTPTimeoutException( |
345 'Timed out waiting for CONNECT response from proxy') | 359 'Timed out waiting for CONNECT response from proxy') |
346 while not r.complete(): | 360 while not r.complete(): |
347 try: | 361 try: |
362 # We're a friend of the response class, so let | |
363 # us use the private attribute. | |
364 # pylint: disable=W0212 | |
348 if not r._select(): | 365 if not r._select(): |
349 if not r.complete(): | 366 if not r.complete(): |
350 raise timeout_exc | 367 raise timeout_exc |
351 except HTTPTimeoutException: | 368 except HTTPTimeoutException: |
352 # This raise/except pattern looks goofy, but | 369 # This raise/except pattern looks goofy, but |
374 if self._ssl_validator: | 391 if self._ssl_validator: |
375 self._ssl_validator(sock) | 392 self._ssl_validator(sock) |
376 sock.setblocking(0) | 393 sock.setblocking(0) |
377 self.sock = sock | 394 self.sock = sock |
378 | 395 |
379 def buildheaders(self, method, path, headers, http_ver): | 396 def _buildheaders(self, method, path, headers, http_ver): |
380 if self.ssl and self.port == 443 or self.port == 80: | 397 if self.ssl and self.port == 443 or self.port == 80: |
381 # default port for protocol, so leave it out | 398 # default port for protocol, so leave it out |
382 hdrhost = self.host | 399 hdrhost = self.host |
383 else: | 400 else: |
384 # include nonstandard port in header | 401 # include nonstandard port in header |
435 self._current_response = None | 452 self._current_response = None |
436 return False | 453 return False |
437 return True | 454 return True |
438 return False | 455 return False |
439 | 456 |
457 def _reconnect(self, where): | |
458 logger.info('reconnecting during %s', where) | |
459 self.close() | |
460 self._connect() | |
461 | |
440 def request(self, method, path, body=None, headers={}, | 462 def request(self, method, path, body=None, headers={}, |
441 expect_continue=False): | 463 expect_continue=False): |
442 """Send a request to the server. | 464 """Send a request to the server. |
443 | 465 |
444 For increased flexibility, this does not return the response | 466 For increased flexibility, this does not return the response |
472 chunked = True | 494 chunked = True |
473 else: | 495 else: |
474 raise BadRequestData('body has no __len__() nor read()') | 496 raise BadRequestData('body has no __len__() nor read()') |
475 | 497 |
476 self._connect() | 498 self._connect() |
477 outgoing_headers = self.buildheaders( | 499 outgoing_headers = self._buildheaders( |
478 method, path, hdrs, self.http_version) | 500 method, path, hdrs, self.http_version) |
479 response = None | 501 response = None |
480 first = True | 502 first = True |
481 | |
482 def reconnect(where): | |
483 logger.info('reconnecting during %s', where) | |
484 self.close() | |
485 self._connect() | |
486 | 503 |
487 while ((outgoing_headers or body) | 504 while ((outgoing_headers or body) |
488 and not (response and response.complete())): | 505 and not (response and response.complete())): |
489 select_timeout = self.timeout | 506 select_timeout = self.timeout |
490 out = outgoing_headers or body | 507 out = outgoing_headers or body |
521 try: | 538 try: |
522 data = r[0].recv(INCOMING_BUFFER_SIZE) | 539 data = r[0].recv(INCOMING_BUFFER_SIZE) |
523 except socket.sslerror, e: | 540 except socket.sslerror, e: |
524 if e.args[0] != socket.SSL_ERROR_WANT_READ: | 541 if e.args[0] != socket.SSL_ERROR_WANT_READ: |
525 raise | 542 raise |
526 logger.debug( | 543 logger.debug('SSL_ERROR_WANT_READ while sending ' |
527 'SSL_ERROR_WANT_READ while sending data, retrying...') | 544 'data, retrying...') |
528 continue | 545 continue |
529 if not data: | 546 if not data: |
530 logger.info('socket appears closed in read') | 547 logger.info('socket appears closed in read') |
531 self.sock = None | 548 self.sock = None |
532 self._current_response = None | 549 self._current_response = None |
533 if response is not None: | 550 if response is not None: |
551 # We're a friend of the response class, so let | |
552 # us use the private attribute. | |
553 # pylint: disable=W0212 | |
534 response._close() | 554 response._close() |
535 # This if/elif ladder is a bit subtle, | 555 # This if/elif ladder is a bit subtle, |
536 # comments in each branch should help. | 556 # comments in each branch should help. |
537 if response is not None and response.complete(): | 557 if response is not None and response.complete(): |
538 # Server responded completely and then | 558 # Server responded completely and then |
548 # after getting a really large response | 568 # after getting a really large response |
549 # from the server. | 569 # from the server. |
550 logger.info( | 570 logger.info( |
551 'Connection appeared closed in read on first' | 571 'Connection appeared closed in read on first' |
552 ' request loop iteration, will retry.') | 572 ' request loop iteration, will retry.') |
553 reconnect('read') | 573 self._reconnect('read') |
554 continue | 574 continue |
555 else: | 575 else: |
556 # We didn't just send the first data hunk, | 576 # We didn't just send the first data hunk, |
557 # and either have a partial response or no | 577 # and either have a partial response or no |
558 # response at all. There's really nothing | 578 # response at all. There's really nothing |
561 'Connection appears closed after ' | 581 'Connection appears closed after ' |
562 'some request data was written, but the ' | 582 'some request data was written, but the ' |
563 'response was missing or incomplete!') | 583 'response was missing or incomplete!') |
564 logger.debug('read %d bytes in request()', len(data)) | 584 logger.debug('read %d bytes in request()', len(data)) |
565 if response is None: | 585 if response is None: |
566 response = self.response_class(r[0], self.timeout, method) | 586 response = self.response_class( |
587 r[0], self.timeout, method) | |
588 # We're a friend of the response class, so let us | |
589 # use the private attribute. | |
590 # pylint: disable=W0212 | |
567 response._load_response(data) | 591 response._load_response(data) |
568 # Jump to the next select() call so we load more | 592 # Jump to the next select() call so we load more |
569 # data if the server is still sending us content. | 593 # data if the server is still sending us content. |
570 continue | 594 continue |
571 except socket.error, e: | 595 except socket.error, e: |
574 | 598 |
575 # outgoing data | 599 # outgoing data |
576 if w and out: | 600 if w and out: |
577 try: | 601 try: |
578 if getattr(out, 'read', False): | 602 if getattr(out, 'read', False): |
603 # pylint guesses the type of out incorrectly here | |
604 # pylint: disable=E1103 | |
579 data = out.read(OUTGOING_BUFFER_SIZE) | 605 data = out.read(OUTGOING_BUFFER_SIZE) |
580 if not data: | 606 if not data: |
581 continue | 607 continue |
582 if len(data) < OUTGOING_BUFFER_SIZE: | 608 if len(data) < OUTGOING_BUFFER_SIZE: |
583 if chunked: | 609 if chunked: |
597 # similar to selecting on a raw socket. | 623 # similar to selecting on a raw socket. |
598 continue | 624 continue |
599 elif (e[0] not in (errno.ECONNRESET, errno.EPIPE) | 625 elif (e[0] not in (errno.ECONNRESET, errno.EPIPE) |
600 and not first): | 626 and not first): |
601 raise | 627 raise |
602 reconnect('write') | 628 self._reconnect('write') |
603 amt = self.sock.send(out) | 629 amt = self.sock.send(out) |
604 logger.debug('sent %d', amt) | 630 logger.debug('sent %d', amt) |
605 first = False | 631 first = False |
606 # stash data we think we sent in case the socket breaks | |
607 # when we read from it | |
608 if was_first: | |
609 sent_data = out[:amt] | |
610 if out is body: | 632 if out is body: |
611 body = out[amt:] | 633 body = out[amt:] |
612 else: | 634 else: |
613 outgoing_headers = out[amt:] | 635 outgoing_headers = out[amt:] |
614 | 636 |
615 # close if the server response said to or responded before eating | 637 # close if the server response said to or responded before eating |
616 # the whole request | 638 # the whole request |
617 if response is None: | 639 if response is None: |
618 response = self.response_class(self.sock, self.timeout, method) | 640 response = self.response_class(self.sock, self.timeout, method) |
619 complete = response.complete() | |
620 data_left = bool(outgoing_headers or body) | 641 data_left = bool(outgoing_headers or body) |
621 if data_left: | 642 if data_left: |
622 logger.info('stopped sending request early, ' | 643 logger.info('stopped sending request early, ' |
623 'will close the socket to be safe.') | 644 'will close the socket to be safe.') |
624 response.will_close = True | 645 response.will_close = True |
627 # the socket | 648 # the socket |
628 self.sock = None | 649 self.sock = None |
629 self._current_response = response | 650 self._current_response = response |
630 | 651 |
631 def getresponse(self): | 652 def getresponse(self): |
653 """Returns the response to the most recent request.""" | |
632 if self._current_response is None: | 654 if self._current_response is None: |
633 raise httplib.ResponseNotReady() | 655 raise httplib.ResponseNotReady() |
634 r = self._current_response | 656 r = self._current_response |
635 while r.headers is None: | 657 while r.headers is None: |
658 # We're a friend of the response class, so let us use the | |
659 # private attribute. | |
660 # pylint: disable=W0212 | |
636 if not r._select() and not r.complete(): | 661 if not r._select() and not r.complete(): |
637 raise _readers.HTTPRemoteClosedError() | 662 raise _readers.HTTPRemoteClosedError() |
638 if r.will_close: | 663 if r.will_close: |
639 self.sock = None | 664 self.sock = None |
640 self._current_response = None | 665 self._current_response = None |