378 pass |
378 pass |
379 |
379 |
380 |
380 |
381 class HTTPResponse(httplib.HTTPResponse): |
381 class HTTPResponse(httplib.HTTPResponse): |
382 # we need to subclass HTTPResponse in order to |
382 # we need to subclass HTTPResponse in order to |
383 # 1) add readline(), readlines(), and readinto() methods |
383 # 1) add close_connection() methods |
384 # 2) add close_connection() methods |
384 # 2) add info() and geturl() methods |
385 # 3) add info() and geturl() methods |
385 # 3) add accounting for read(), readlines() and readinto() |
386 |
|
387 # in order to add readline(), read must be modified to deal with a |
|
388 # buffer. example: readline must read a buffer and then spit back |
|
389 # one line at a time. The only real alternative is to read one |
|
390 # BYTE at a time (ick). Once something has been read, it can't be |
|
391 # put back (ok, maybe it can, but that's even uglier than this), |
|
392 # so if you THEN do a normal read, you must first take stuff from |
|
393 # the buffer. |
|
394 |
|
395 # the read method wraps the original to accommodate buffering, |
|
396 # although read() never adds to the buffer. |
|
397 # Both readline and readlines have been stolen with almost no |
|
398 # modification from socket.py |
|
399 |
386 |
400 def __init__(self, sock, debuglevel=0, strict=0, method=None): |
387 def __init__(self, sock, debuglevel=0, strict=0, method=None): |
401 httplib.HTTPResponse.__init__( |
388 httplib.HTTPResponse.__init__( |
402 self, sock, debuglevel=debuglevel, method=method |
389 self, sock, debuglevel=debuglevel, method=method |
403 ) |
390 ) |
440 |
424 |
441 def geturl(self): |
425 def geturl(self): |
442 return self._url |
426 return self._url |
443 |
427 |
444 def read(self, amt=None): |
428 def read(self, amt=None): |
445 # the _rbuf test is only in this first if for speed. It's not |
429 data = super().read(amt) |
446 # logically necessary |
|
447 if self._rbuf and amt is not None: |
|
448 L = len(self._rbuf) |
|
449 if amt > L: |
|
450 amt -= L |
|
451 else: |
|
452 s = self._rbuf[:amt] |
|
453 self._rbuf = self._rbuf[amt:] |
|
454 return s |
|
455 # Careful! http.client.HTTPResponse.read() on Python 3 is |
|
456 # implemented using readinto(), which can duplicate self._rbuf |
|
457 # if it's not empty. |
|
458 s = self._rbuf |
|
459 self._rbuf = b'' |
|
460 data = self._raw_read(amt) |
|
461 |
|
462 self.receivedbytescount += len(data) |
430 self.receivedbytescount += len(data) |
463 try: |
431 try: |
464 self._connection.receivedbytescount += len(data) |
432 self._connection.receivedbytescount += len(data) |
465 except AttributeError: |
433 except AttributeError: |
466 pass |
434 pass |
467 try: |
435 try: |
468 self._handler.parent.receivedbytescount += len(data) |
436 self._handler.parent.receivedbytescount += len(data) |
469 except AttributeError: |
437 except AttributeError: |
470 pass |
438 pass |
471 |
439 return data |
472 s += data |
|
473 return s |
|
474 |
|
475 # stolen from Python SVN #68532 to fix issue1088 |
|
476 def _read_chunked(self, amt): |
|
477 chunk_left = self.chunk_left |
|
478 parts = [] |
|
479 |
|
480 while True: |
|
481 if chunk_left is None: |
|
482 line = self.fp.readline() |
|
483 i = line.find(b';') |
|
484 if i >= 0: |
|
485 line = line[:i] # strip chunk-extensions |
|
486 try: |
|
487 chunk_left = int(line, 16) |
|
488 except ValueError: |
|
489 # close the connection as protocol synchronization is |
|
490 # probably lost |
|
491 self.close() |
|
492 raise httplib.IncompleteRead(b''.join(parts)) |
|
493 if chunk_left == 0: |
|
494 break |
|
495 if amt is None: |
|
496 parts.append(self._safe_read(chunk_left)) |
|
497 elif amt < chunk_left: |
|
498 parts.append(self._safe_read(amt)) |
|
499 self.chunk_left = chunk_left - amt |
|
500 return b''.join(parts) |
|
501 elif amt == chunk_left: |
|
502 parts.append(self._safe_read(amt)) |
|
503 self._safe_read(2) # toss the CRLF at the end of the chunk |
|
504 self.chunk_left = None |
|
505 return b''.join(parts) |
|
506 else: |
|
507 parts.append(self._safe_read(chunk_left)) |
|
508 amt -= chunk_left |
|
509 |
|
510 # we read the whole chunk, get another |
|
511 self._safe_read(2) # toss the CRLF at the end of the chunk |
|
512 chunk_left = None |
|
513 |
|
514 # read and discard trailer up to the CRLF terminator |
|
515 ### note: we shouldn't have any trailers! |
|
516 while True: |
|
517 line = self.fp.readline() |
|
518 if not line: |
|
519 # a vanishingly small number of sites EOF without |
|
520 # sending the trailer |
|
521 break |
|
522 if line == b'\r\n': |
|
523 break |
|
524 |
|
525 # we read everything; close the "file" |
|
526 self.close() |
|
527 |
|
528 return b''.join(parts) |
|
529 |
440 |
530 def readline(self): |
441 def readline(self): |
531 # Fast path for a line is already available in read buffer. |
442 data = super().readline() |
532 i = self._rbuf.find(b'\n') |
443 self.receivedbytescount += len(data) |
533 if i >= 0: |
444 try: |
534 i += 1 |
445 self._connection.receivedbytescount += len(data) |
535 line = self._rbuf[:i] |
|
536 self._rbuf = self._rbuf[i:] |
|
537 return line |
|
538 |
|
539 # No newline in local buffer. Read until we find one. |
|
540 # readinto read via readinto will already return _rbuf |
|
541 if self._raw_readinto is None: |
|
542 chunks = [self._rbuf] |
|
543 else: |
|
544 chunks = [] |
|
545 i = -1 |
|
546 readsize = self._rbufsize |
|
547 while True: |
|
548 new = self._raw_read(readsize) |
|
549 if not new: |
|
550 break |
|
551 |
|
552 self.receivedbytescount += len(new) |
|
553 self._connection.receivedbytescount += len(new) |
|
554 try: |
|
555 self._handler.parent.receivedbytescount += len(new) |
|
556 except AttributeError: |
|
557 pass |
|
558 |
|
559 chunks.append(new) |
|
560 i = new.find(b'\n') |
|
561 if i >= 0: |
|
562 break |
|
563 |
|
564 # We either have exhausted the stream or have a newline in chunks[-1]. |
|
565 |
|
566 # EOF |
|
567 if i == -1: |
|
568 self._rbuf = b'' |
|
569 return b''.join(chunks) |
|
570 |
|
571 i += 1 |
|
572 self._rbuf = chunks[-1][i:] |
|
573 chunks[-1] = chunks[-1][:i] |
|
574 return b''.join(chunks) |
|
575 |
|
576 def readinto(self, dest): |
|
577 if self._raw_readinto is None: |
|
578 res = self.read(len(dest)) |
|
579 if not res: |
|
580 return 0 |
|
581 dest[0 : len(res)] = res |
|
582 return len(res) |
|
583 total = len(dest) |
|
584 have = len(self._rbuf) |
|
585 if have >= total: |
|
586 dest[0:total] = self._rbuf[:total] |
|
587 self._rbuf = self._rbuf[total:] |
|
588 return total |
|
589 mv = memoryview(dest) |
|
590 got = self._raw_readinto(mv[have:total]) |
|
591 |
|
592 self.receivedbytescount += got |
|
593 self._connection.receivedbytescount += got |
|
594 try: |
|
595 self._handler.receivedbytescount += got |
|
596 except AttributeError: |
446 except AttributeError: |
597 pass |
447 pass |
598 |
448 try: |
599 dest[0:have] = self._rbuf |
449 self._handler.parent.receivedbytescount += len(data) |
600 got += len(self._rbuf) |
450 except AttributeError: |
601 self._rbuf = b'' |
451 pass |
|
452 return data |
|
453 |
|
454 def readinto(self, dest): |
|
455 got = super().readinto(dest) |
|
456 self.receivedbytescount += got |
|
457 try: |
|
458 self._connection.receivedbytescount += got |
|
459 except AttributeError: |
|
460 pass |
|
461 try: |
|
462 self._handler.parent.receivedbytescount += got |
|
463 except AttributeError: |
|
464 pass |
602 return got |
465 return got |
603 |
466 |
604 |
467 |
605 def safesend(self, str): |
468 def safesend(self, str): |
606 """Send `str' to the server. |
469 """Send `str' to the server. |