Mercurial > hg
annotate mercurial/keepalive.py @ 43117:8ff1ecfadcd1
cleanup: join string literals that are already on one line
Thanks to Kyle for noticing this and for providing the regular
expression to run on the codebase.
This patch has been reviewed by the test suite and they approved of
it.
# skip-blame: fallout from mass reformatting
Differential Revision: https://phab.mercurial-scm.org/D7028
author | Martin von Zweigbergk <martinvonz@google.com> |
---|---|
date | Tue, 08 Oct 2019 15:06:18 -0700 |
parents | d783f945a701 |
children | 9f70512ae2cf |
rev | line source |
---|---|
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
1 # This library is free software; you can redistribute it and/or |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
2 # modify it under the terms of the GNU Lesser General Public |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
3 # License as published by the Free Software Foundation; either |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
4 # version 2.1 of the License, or (at your option) any later version. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
5 # |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
6 # This library is distributed in the hope that it will be useful, |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
7 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
9 # Lesser General Public License for more details. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
10 # |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
11 # You should have received a copy of the GNU Lesser General Public |
15782
7de7630053cb
Remove FSF mailing address from GPL headers
Martin Geisler <mg@aragost.com>
parents:
14958
diff
changeset
|
12 # License along with this library; if not, see |
7de7630053cb
Remove FSF mailing address from GPL headers
Martin Geisler <mg@aragost.com>
parents:
14958
diff
changeset
|
13 # <http://www.gnu.org/licenses/>. |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
14 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
15 # This file is part of urlgrabber, a high-level cross-protocol url-grabber |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
16 # Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
17 |
4026
8520a773a141
fix for digest auth when using keepalive.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
2600
diff
changeset
|
18 # Modified by Benoit Boissinot: |
8520a773a141
fix for digest auth when using keepalive.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
2600
diff
changeset
|
19 # - fix for digest auth (inspired from urllib2.py @ Python v2.4) |
6470
ac0bcd951c2c
python 2.6 compatibility: compatibility wrappers for hash functions
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents:
6001
diff
changeset
|
20 # Modified by Dirkjan Ochtman: |
ac0bcd951c2c
python 2.6 compatibility: compatibility wrappers for hash functions
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents:
6001
diff
changeset
|
21 # - import md5 function from a local util module |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
22 # Modified by Augie Fackler: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
23 # - add safesend method and use it to prevent broken pipe errors |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
24 # on large POST requests |
4026
8520a773a141
fix for digest auth when using keepalive.py
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
2600
diff
changeset
|
25 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
26 """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
27 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
28 >>> import urllib2 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
29 >>> from keepalive import HTTPHandler |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
30 >>> keepalive_handler = HTTPHandler() |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
31 >>> opener = urlreq.buildopener(keepalive_handler) |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
32 >>> urlreq.installopener(opener) |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
33 >>> |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
34 >>> fo = urlreq.urlopen('http://www.python.org') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
35 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
36 If a connection to a given host is requested, and all of the existing |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
37 connections are still in use, another connection will be opened. If |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
38 the handler tries to use an existing connection but it fails in some |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
39 way, it will be closed and removed from the pool. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
40 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
41 To remove the handler, simply re-run build_opener with no arguments, and |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
42 install that opener. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
43 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
44 You can explicitly close connections by using the close_connection() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
45 method of the returned file-like object (described below) or you can |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
46 use the handler methods: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
47 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
48 close_connection(host) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
49 close_all() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
50 open_connections() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
51 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
52 NOTE: using the close_connection and close_all methods of the handler |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
53 should be done with care when using multiple threads. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
54 * there is nothing that prevents another thread from creating new |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
55 connections immediately after connections are closed |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
56 * no checks are done to prevent in-use connections from being closed |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
57 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
58 >>> keepalive_handler.close_all() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
59 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
60 EXTRA ATTRIBUTES AND METHODS |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
61 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
62 Upon a status of 200, the object returned has a few additional |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
63 attributes and methods, which should not be used if you want to |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
64 remain consistent with the normal urllib2-returned objects: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
65 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
66 close_connection() - close the connection to the host |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
67 readlines() - you know, readlines() |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17251
diff
changeset
|
68 status - the return status (i.e. 404) |
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17251
diff
changeset
|
69 reason - english translation of status (i.e. 'File not found') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
70 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
71 If you want the best of both worlds, use this inside an |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
72 AttributeError-catching try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
73 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
74 >>> try: status = fo.status |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
75 >>> except AttributeError: status = None |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
76 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
77 Unfortunately, these are ONLY there if status == 200, so it's not |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
78 easy to distinguish between non-200 responses. The reason is that |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
79 urllib2 tries to do clever things with error codes 301, 302, 401, |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
80 and 407, and it wraps the object upon return. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
81 """ |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
82 |
2444
5eb02f9ed804
Update keepalive.py to current CVS version of urlgrabber.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
2435
diff
changeset
|
83 # $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $ |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
84 |
27616
0765d8423fbc
keepalive: use print function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27507
diff
changeset
|
85 from __future__ import absolute_import, print_function |
27507
a16489f9132d
keepalive: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
86 |
41408
a43acfa2b76d
keepalive: use collections.defaultdict for host map
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40874
diff
changeset
|
87 import collections |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
88 import errno |
29341
0d83ad967bf8
cleanup: replace uses of util.(md5|sha1|sha256|sha512) with hashlib.\1
Augie Fackler <raf@durin42.com>
parents:
28883
diff
changeset
|
89 import hashlib |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
90 import socket |
27507
a16489f9132d
keepalive: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
91 import sys |
29456
e61d384e3be4
keepalive: switch from thread to threading module
Pulkit Goyal <7895pulkit@gmail.com>
parents:
29455
diff
changeset
|
92 import threading |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
93 |
34308
9bd003052d55
keepalive: add more context to bad status line errors
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32668
diff
changeset
|
94 from .i18n import _ |
43089
c59eb1560c44
py3: manually import getattr where it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43077
diff
changeset
|
95 from .pycompat import getattr |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
96 from . import ( |
35582
72b91f905065
py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35364
diff
changeset
|
97 node, |
34427
a454123f5d94
keepalive: python 3 portability tweaks
Augie Fackler <augie@google.com>
parents:
34331
diff
changeset
|
98 pycompat, |
34466
1232f7fa00c3
cleanup: use urllibcompat for renamed methods on urllib request objects
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
99 urllibcompat, |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
100 util, |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
101 ) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
102 from .utils import procutil |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
103 |
29455
0c741fd6158a
py3: conditionalize httplib import
Pulkit Goyal <7895pulkit@gmail.com>
parents:
29341
diff
changeset
|
104 httplib = util.httplib |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
105 urlerr = util.urlerr |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
106 urlreq = util.urlreq |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
107 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
108 DEBUG = None |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
109 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
110 |
14764
a7d5816087a9
classes: fix class style problems found by b071cd58af50
Thomas Arendsen Hein <thomas@intevation.de>
parents:
14494
diff
changeset
|
111 class ConnectionManager(object): |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
112 """ |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
113 The connection manager must be able to: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
114 * keep track of all existing |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
115 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
116 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
117 def __init__(self): |
29456
e61d384e3be4
keepalive: switch from thread to threading module
Pulkit Goyal <7895pulkit@gmail.com>
parents:
29455
diff
changeset
|
118 self._lock = threading.Lock() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
119 self._hostmap = collections.defaultdict(list) # host -> [connection] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
120 self._connmap = {} # map connections to host |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
121 self._readymap = {} # map connection to ready state |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
122 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
123 def add(self, host, connection, ready): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
124 self._lock.acquire() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
125 try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
126 self._hostmap[host].append(connection) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
127 self._connmap[connection] = host |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
128 self._readymap[connection] = ready |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
129 finally: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
130 self._lock.release() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
131 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
132 def remove(self, connection): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
133 self._lock.acquire() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
134 try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
135 try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
136 host = self._connmap[connection] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
137 except KeyError: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
138 pass |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
139 else: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
140 del self._connmap[connection] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
141 del self._readymap[connection] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
142 self._hostmap[host].remove(connection) |
34435
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34427
diff
changeset
|
143 if not self._hostmap[host]: |
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34427
diff
changeset
|
144 del self._hostmap[host] |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
145 finally: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
146 self._lock.release() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
147 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
148 def set_ready(self, connection, ready): |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
149 try: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
150 self._readymap[connection] = ready |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
151 except KeyError: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
152 pass |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
153 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
154 def get_ready_conn(self, host): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
155 conn = None |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
156 self._lock.acquire() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
157 try: |
41408
a43acfa2b76d
keepalive: use collections.defaultdict for host map
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40874
diff
changeset
|
158 for c in self._hostmap[host]: |
a43acfa2b76d
keepalive: use collections.defaultdict for host map
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40874
diff
changeset
|
159 if self._readymap[c]: |
41409
1db94ebbc207
keepalive: track ready state with a bool
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41408
diff
changeset
|
160 self._readymap[c] = False |
41408
a43acfa2b76d
keepalive: use collections.defaultdict for host map
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40874
diff
changeset
|
161 conn = c |
a43acfa2b76d
keepalive: use collections.defaultdict for host map
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40874
diff
changeset
|
162 break |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
163 finally: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
164 self._lock.release() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
165 return conn |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
166 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
167 def get_all(self, host=None): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
168 if host: |
41408
a43acfa2b76d
keepalive: use collections.defaultdict for host map
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40874
diff
changeset
|
169 return list(self._hostmap[host]) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
170 else: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
171 return dict(self._hostmap) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
172 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
173 |
14764
a7d5816087a9
classes: fix class style problems found by b071cd58af50
Thomas Arendsen Hein <thomas@intevation.de>
parents:
14494
diff
changeset
|
174 class KeepAliveHandler(object): |
40043
6509fcec830c
url: allow to configure timeout on http connection
Cédric Krier <ced@b2ck.com>
parents:
40033
diff
changeset
|
175 def __init__(self, timeout=None): |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
176 self._cm = ConnectionManager() |
40043
6509fcec830c
url: allow to configure timeout on http connection
Cédric Krier <ced@b2ck.com>
parents:
40033
diff
changeset
|
177 self._timeout = timeout |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
178 self.requestscount = 0 |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
179 self.sentbytescount = 0 |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
180 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
181 #### Connection Management |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
182 def open_connections(self): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
183 """return a list of connected hosts and the number of connections |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
184 to each. [('foo.com:80', 2), ('bar.org', 1)]""" |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
185 return [(host, len(li)) for (host, li) in self._cm.get_all().items()] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
186 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
187 def close_connection(self, host): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
188 """close connection(s) to <host> |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
189 host is the host:port spec, as in 'www.cnn.com:8080' as passed in. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
190 no error occurs if there is no connection to that host.""" |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
191 for h in self._cm.get_all(host): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
192 self._cm.remove(h) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
193 h.close() |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
194 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
195 def close_all(self): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
196 """close all open connections""" |
43106
d783f945a701
py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43103
diff
changeset
|
197 for host, conns in pycompat.iteritems(self._cm.get_all()): |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
198 for h in conns: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
199 self._cm.remove(h) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
200 h.close() |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
201 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
202 def _request_closed(self, request, host, connection): |
17251
98166640b356
help: fix some instances of 'the the'
Mads Kiilerich <mads@kiilerich.com>
parents:
16705
diff
changeset
|
203 """tells us that this request is now closed and that the |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
204 connection is ready for another request""" |
41409
1db94ebbc207
keepalive: track ready state with a bool
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41408
diff
changeset
|
205 self._cm.set_ready(connection, True) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
206 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
207 def _remove_connection(self, host, connection, close=0): |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
208 if close: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
209 connection.close() |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
210 self._cm.remove(connection) |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
211 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
212 #### Transaction Execution |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
213 def http_open(self, req): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
214 return self.do_open(HTTPConnection, req) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
215 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
216 def do_open(self, http_class, req): |
34466
1232f7fa00c3
cleanup: use urllibcompat for renamed methods on urllib request objects
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
217 host = urllibcompat.gethost(req) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
218 if not host: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
219 raise urlerr.urlerror(b'no host given') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
220 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
221 try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
222 h = self._cm.get_ready_conn(host) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
223 while h: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
224 r = self._reuse_connection(h, req, host) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
225 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
226 # if this response is non-None, then it worked and we're |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
227 # done. Break out, skipping the else block. |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
228 if r: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
229 break |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
230 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
231 # connection is bad - possibly closed by server |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
232 # discard it and ask for the next free connection |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
233 h.close() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
234 self._cm.remove(h) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
235 h = self._cm.get_ready_conn(host) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
236 else: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
237 # no (working) free connections were found. Create a new one. |
40043
6509fcec830c
url: allow to configure timeout on http connection
Cédric Krier <ced@b2ck.com>
parents:
40033
diff
changeset
|
238 h = http_class(host, timeout=self._timeout) |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
239 if DEBUG: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
240 DEBUG.info( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
241 b"creating new connection to %s (%d)", host, id(h) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
242 ) |
41409
1db94ebbc207
keepalive: track ready state with a bool
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41408
diff
changeset
|
243 self._cm.add(host, h, False) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
244 self._start_transaction(h, req) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
245 r = h.getresponse() |
34308
9bd003052d55
keepalive: add more context to bad status line errors
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32668
diff
changeset
|
246 # The string form of BadStatusLine is the status line. Add some context |
9bd003052d55
keepalive: add more context to bad status line errors
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32668
diff
changeset
|
247 # to make the error message slightly more useful. |
9bd003052d55
keepalive: add more context to bad status line errors
Gregory Szorc <gregory.szorc@gmail.com>
parents:
32668
diff
changeset
|
248 except httplib.BadStatusLine as err: |
34427
a454123f5d94
keepalive: python 3 portability tweaks
Augie Fackler <augie@google.com>
parents:
34331
diff
changeset
|
249 raise urlerr.urlerror( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
250 _(b'bad HTTP status line: %s') % pycompat.sysbytes(err.line) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
251 ) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
22958
diff
changeset
|
252 except (socket.error, httplib.HTTPException) as err: |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
253 raise urlerr.urlerror(err) |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
254 |
39649
d6d094259d9c
keepalive: work around slight deficiency in vcr
Augie Fackler <raf@durin42.com>
parents:
37688
diff
changeset
|
255 # If not a persistent connection, don't try to reuse it. Look |
d6d094259d9c
keepalive: work around slight deficiency in vcr
Augie Fackler <raf@durin42.com>
parents:
37688
diff
changeset
|
256 # for this using getattr() since vcr doesn't define this |
d6d094259d9c
keepalive: work around slight deficiency in vcr
Augie Fackler <raf@durin42.com>
parents:
37688
diff
changeset
|
257 # attribute, and in that case always close the connection. |
43103
c95b2f40db7c
py3: stop normalizing 2nd argument of *attr() to unicode
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43089
diff
changeset
|
258 if getattr(r, 'will_close', True): |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
259 self._cm.remove(h) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
260 |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
261 if DEBUG: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
262 DEBUG.info(b"STATUS: %s, %s", r.status, r.reason) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
263 r._handler = self |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
264 r._host = host |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
265 r._url = req.get_full_url() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
266 r._connection = h |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
267 r.code = r.status |
2444
5eb02f9ed804
Update keepalive.py to current CVS version of urlgrabber.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
2435
diff
changeset
|
268 r.headers = r.msg |
5eb02f9ed804
Update keepalive.py to current CVS version of urlgrabber.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
2435
diff
changeset
|
269 r.msg = r.reason |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
270 |
30487
88a448a12ae8
keepalive: discard legacy Python support for error handling
Augie Fackler <augie@google.com>
parents:
30473
diff
changeset
|
271 return r |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
272 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
273 def _reuse_connection(self, h, req, host): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
274 """start the transaction with a re-used connection |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
275 return a response object (r) upon success or None on failure. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
276 This DOES not close or remove bad connections in cases where |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
277 it returns. However, if an unexpected exception occurs, it |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
278 will close and remove the connection before re-raising. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
279 """ |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
280 try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
281 self._start_transaction(h, req) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
282 r = h.getresponse() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
283 # note: just because we got something back doesn't mean it |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
284 # worked. We'll check the version below, too. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
285 except (socket.error, httplib.HTTPException): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
286 r = None |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
287 except: # re-raises |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
288 # adding this block just in case we've missed |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
289 # something we will still raise the exception, but |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
290 # lets try and close the connection and remove it |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
291 # first. We previously got into a nasty loop |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
292 # where an exception was uncaught, and so the |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
293 # connection stayed open. On the next try, the |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17251
diff
changeset
|
294 # same exception was raised, etc. The trade-off is |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
295 # that it's now possible this call will raise |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
296 # a DIFFERENT exception |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
297 if DEBUG: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
298 DEBUG.error( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43106
diff
changeset
|
299 b"unexpected exception - closing connection to %s (%d)", |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
300 host, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
301 id(h), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
302 ) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
303 self._cm.remove(h) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
304 h.close() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
305 raise |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
306 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
307 if r is None or r.version == 9: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
308 # httplib falls back to assuming HTTP 0.9 if it gets a |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
309 # bad header back. This is most likely to happen if |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
310 # the socket has been closed by the server since we |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
311 # last used the connection. |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
312 if DEBUG: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
313 DEBUG.info( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
314 b"failed to re-use connection to %s (%d)", host, id(h) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
315 ) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
316 r = None |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
317 else: |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
318 if DEBUG: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
319 DEBUG.info(b"re-using connection to %s (%d)", host, id(h)) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
320 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
321 return r |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
322 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
323 def _start_transaction(self, h, req): |
40379
41506e3b04ee
keepalive: use getattr to avoid AttributeErrors when vcr is in use
Augie Fackler <augie@google.com>
parents:
40043
diff
changeset
|
324 oldbytescount = getattr(h, 'sentbytescount', 0) |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
325 |
8233
655c435efe92
keepalive: fix 4f13ed6ee544, reintroduce unredirected_hdrs
Patrick Mezard <pmezard@gmail.com>
parents:
8146
diff
changeset
|
326 # What follows mostly reimplements HTTPConnection.request() |
31999
aa836f56c3cc
keepalive: send HTTP request headers in a deterministic order
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30922
diff
changeset
|
327 # except it adds self.parent.addheaders in the mix and sends headers |
aa836f56c3cc
keepalive: send HTTP request headers in a deterministic order
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30922
diff
changeset
|
328 # in a deterministic order (to make testing easier). |
aa836f56c3cc
keepalive: send HTTP request headers in a deterministic order
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30922
diff
changeset
|
329 headers = util.sortdict(self.parent.addheaders) |
aa836f56c3cc
keepalive: send HTTP request headers in a deterministic order
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30922
diff
changeset
|
330 headers.update(sorted(req.headers.items())) |
aa836f56c3cc
keepalive: send HTTP request headers in a deterministic order
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30922
diff
changeset
|
331 headers.update(sorted(req.unredirected_hdrs.items())) |
aa836f56c3cc
keepalive: send HTTP request headers in a deterministic order
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30922
diff
changeset
|
332 headers = util.sortdict((n.lower(), v) for n, v in headers.items()) |
8233
655c435efe92
keepalive: fix 4f13ed6ee544, reintroduce unredirected_hdrs
Patrick Mezard <pmezard@gmail.com>
parents:
8146
diff
changeset
|
333 skipheaders = {} |
37665
83250442dc81
py3: use str variables to check keys in request header
Pulkit Goyal <7895pulkit@gmail.com>
parents:
37587
diff
changeset
|
334 for n in (r'host', r'accept-encoding'): |
8233
655c435efe92
keepalive: fix 4f13ed6ee544, reintroduce unredirected_hdrs
Patrick Mezard <pmezard@gmail.com>
parents:
8146
diff
changeset
|
335 if n in headers: |
37665
83250442dc81
py3: use str variables to check keys in request header
Pulkit Goyal <7895pulkit@gmail.com>
parents:
37587
diff
changeset
|
336 skipheaders[r'skip_' + n.replace(r'-', r'_')] = 1 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
337 try: |
34466
1232f7fa00c3
cleanup: use urllibcompat for renamed methods on urllib request objects
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
338 if urllibcompat.hasdata(req): |
1232f7fa00c3
cleanup: use urllibcompat for renamed methods on urllib request objects
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
339 data = urllibcompat.getdata(req) |
30922
1beeb5185930
keepalive: honor urllib2 style get_method overrides
John Mulligan <phlogistonjohn@asynchrono.us>
parents:
30688
diff
changeset
|
340 h.putrequest( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
341 req.get_method(), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
342 urllibcompat.getselector(req), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
343 **skipheaders |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
344 ) |
36295
19a04ca90413
keepalive: headers are native strings, mark them as such
Augie Fackler <augie@google.com>
parents:
35582
diff
changeset
|
345 if r'content-type' not in headers: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
346 h.putheader( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
347 r'Content-type', r'application/x-www-form-urlencoded' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
348 ) |
36295
19a04ca90413
keepalive: headers are native strings, mark them as such
Augie Fackler <augie@google.com>
parents:
35582
diff
changeset
|
349 if r'content-length' not in headers: |
19a04ca90413
keepalive: headers are native strings, mark them as such
Augie Fackler <augie@google.com>
parents:
35582
diff
changeset
|
350 h.putheader(r'Content-length', r'%d' % len(data)) |
8146
4f13ed6ee544
keepalive: attempt to fix issue1003
Matt Mackall <mpm@selenic.com>
parents:
7875
diff
changeset
|
351 else: |
30922
1beeb5185930
keepalive: honor urllib2 style get_method overrides
John Mulligan <phlogistonjohn@asynchrono.us>
parents:
30688
diff
changeset
|
352 h.putrequest( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
353 req.get_method(), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
354 urllibcompat.getselector(req), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
355 **skipheaders |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
356 ) |
28278
b1b22185c764
keepalive: remove useless parentheses around exception type
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27616
diff
changeset
|
357 except socket.error as err: |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
358 raise urlerr.urlerror(err) |
8233
655c435efe92
keepalive: fix 4f13ed6ee544, reintroduce unredirected_hdrs
Patrick Mezard <pmezard@gmail.com>
parents:
8146
diff
changeset
|
359 for k, v in headers.items(): |
8146
4f13ed6ee544
keepalive: attempt to fix issue1003
Matt Mackall <mpm@selenic.com>
parents:
7875
diff
changeset
|
360 h.putheader(k, v) |
4f13ed6ee544
keepalive: attempt to fix issue1003
Matt Mackall <mpm@selenic.com>
parents:
7875
diff
changeset
|
361 h.endheaders() |
34466
1232f7fa00c3
cleanup: use urllibcompat for renamed methods on urllib request objects
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
362 if urllibcompat.hasdata(req): |
8146
4f13ed6ee544
keepalive: attempt to fix issue1003
Matt Mackall <mpm@selenic.com>
parents:
7875
diff
changeset
|
363 h.send(data) |
4f13ed6ee544
keepalive: attempt to fix issue1003
Matt Mackall <mpm@selenic.com>
parents:
7875
diff
changeset
|
364 |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
365 # This will fail to record events in case of I/O failure. That's OK. |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
366 self.requestscount += 1 |
40379
41506e3b04ee
keepalive: use getattr to avoid AttributeErrors when vcr is in use
Augie Fackler <augie@google.com>
parents:
40043
diff
changeset
|
367 self.sentbytescount += getattr(h, 'sentbytescount', 0) - oldbytescount |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
368 |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
369 try: |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
370 self.parent.requestscount += 1 |
40379
41506e3b04ee
keepalive: use getattr to avoid AttributeErrors when vcr is in use
Augie Fackler <augie@google.com>
parents:
40043
diff
changeset
|
371 self.parent.sentbytescount += ( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
372 getattr(h, 'sentbytescount', 0) - oldbytescount |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
373 ) |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
374 except AttributeError: |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
375 pass |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
376 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
377 |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
378 class HTTPHandler(KeepAliveHandler, urlreq.httphandler): |
5983
6f1fcbc58efa
httprepo: use separate handlers for HTTP and HTTPS
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4026
diff
changeset
|
379 pass |
6f1fcbc58efa
httprepo: use separate handlers for HTTP and HTTPS
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4026
diff
changeset
|
380 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
381 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
382 class HTTPResponse(httplib.HTTPResponse): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
383 # we need to subclass HTTPResponse in order to |
37297
97eedbd5a56c
keepalive: implement readinto()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37119
diff
changeset
|
384 # 1) add readline(), readlines(), and readinto() methods |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
385 # 2) add close_connection() methods |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
386 # 3) add info() and geturl() methods |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
387 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
388 # in order to add readline(), read must be modified to deal with a |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
389 # buffer. example: readline must read a buffer and then spit back |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
390 # one line at a time. The only real alternative is to read one |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
391 # BYTE at a time (ick). Once something has been read, it can't be |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
392 # put back (ok, maybe it can, but that's even uglier than this), |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
393 # so if you THEN do a normal read, you must first take stuff from |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
394 # the buffer. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
395 |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17251
diff
changeset
|
396 # the read method wraps the original to accommodate buffering, |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
397 # although read() never adds to the buffer. |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
398 # Both readline and readlines have been stolen with almost no |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
399 # modification from socket.py |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
400 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
401 def __init__(self, sock, debuglevel=0, strict=0, method=None): |
34427
a454123f5d94
keepalive: python 3 portability tweaks
Augie Fackler <augie@google.com>
parents:
34331
diff
changeset
|
402 extrakw = {} |
a454123f5d94
keepalive: python 3 portability tweaks
Augie Fackler <augie@google.com>
parents:
34331
diff
changeset
|
403 if not pycompat.ispy3: |
35364
03112a2c9c83
py3: handle keyword arguments correctly in keepalive.py
Pulkit Goyal <7895pulkit@gmail.com>
parents:
34466
diff
changeset
|
404 extrakw[r'strict'] = True |
03112a2c9c83
py3: handle keyword arguments correctly in keepalive.py
Pulkit Goyal <7895pulkit@gmail.com>
parents:
34466
diff
changeset
|
405 extrakw[r'buffering'] = True |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
406 httplib.HTTPResponse.__init__( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
407 self, sock, debuglevel=debuglevel, method=method, **extrakw |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
408 ) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
409 self.fileno = sock.fileno |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
410 self.code = None |
40033
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
411 self.receivedbytescount = 0 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
412 self._rbuf = b'' |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
413 self._rbufsize = 8096 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
414 self._handler = None # inserted by the handler later |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
415 self._host = None # (same) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
416 self._url = None # (same) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
417 self._connection = None # (same) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
418 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
419 _raw_read = httplib.HTTPResponse.read |
37587
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
420 _raw_readinto = getattr(httplib.HTTPResponse, 'readinto', None) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
421 |
41441
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
422 # Python 2.7 has a single close() which closes the socket handle. |
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
423 # This method was effectively renamed to _close_conn() in Python 3. But |
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
424 # there is also a close(). _close_conn() is called by methods like |
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
425 # read(). |
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
426 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
427 def close(self): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
428 if self.fp: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
429 self.fp.close() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
430 self.fp = None |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
431 if self._handler: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
432 self._handler._request_closed( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
433 self, self._host, self._connection |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
434 ) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
435 |
41441
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
436 def _close_conn(self): |
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
437 self.close() |
44d752efdbce
keepalive: implement _close_conn() so closes are known
Gregory Szorc <gregory.szorc@gmail.com>
parents:
41409
diff
changeset
|
438 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
439 def close_connection(self): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
440 self._handler._remove_connection(self._host, self._connection, close=1) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
441 self.close() |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
442 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
443 def info(self): |
2444
5eb02f9ed804
Update keepalive.py to current CVS version of urlgrabber.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
2435
diff
changeset
|
444 return self.headers |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
445 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
446 def geturl(self): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
447 return self._url |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
448 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
449 def read(self, amt=None): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
450 # the _rbuf test is only in this first if for speed. It's not |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
451 # logically necessary |
34331
531332502568
style: always use `x is not None` instead of `not x is None`
Alex Gaynor <agaynor@mozilla.com>
parents:
34308
diff
changeset
|
452 if self._rbuf and amt is not None: |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
453 L = len(self._rbuf) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
454 if amt > L: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
455 amt -= L |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
456 else: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
457 s = self._rbuf[:amt] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
458 self._rbuf = self._rbuf[amt:] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
459 return s |
39816
1cf1680b0554
keepalive: be more careful about self._rbuf when calling super impls
Augie Fackler <augie@google.com>
parents:
39649
diff
changeset
|
460 # Careful! http.client.HTTPResponse.read() on Python 3 is |
1cf1680b0554
keepalive: be more careful about self._rbuf when calling super impls
Augie Fackler <augie@google.com>
parents:
39649
diff
changeset
|
461 # implemented using readinto(), which can duplicate self._rbuf |
1cf1680b0554
keepalive: be more careful about self._rbuf when calling super impls
Augie Fackler <augie@google.com>
parents:
39649
diff
changeset
|
462 # if it's not empty. |
1cf1680b0554
keepalive: be more careful about self._rbuf when calling super impls
Augie Fackler <augie@google.com>
parents:
39649
diff
changeset
|
463 s = self._rbuf |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
464 self._rbuf = b'' |
40033
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
465 data = self._raw_read(amt) |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
466 |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
467 self.receivedbytescount += len(data) |
40426
588f1e9a4d16
http: work around custom http client classes that refuse extra attrs
Augie Fackler <augie@google.com>
parents:
40379
diff
changeset
|
468 try: |
588f1e9a4d16
http: work around custom http client classes that refuse extra attrs
Augie Fackler <augie@google.com>
parents:
40379
diff
changeset
|
469 self._connection.receivedbytescount += len(data) |
588f1e9a4d16
http: work around custom http client classes that refuse extra attrs
Augie Fackler <augie@google.com>
parents:
40379
diff
changeset
|
470 except AttributeError: |
588f1e9a4d16
http: work around custom http client classes that refuse extra attrs
Augie Fackler <augie@google.com>
parents:
40379
diff
changeset
|
471 pass |
40033
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
472 try: |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
473 self._handler.parent.receivedbytescount += len(data) |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
474 except AttributeError: |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
475 pass |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
476 |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
477 s += data |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
478 return s |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
479 |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
480 # stolen from Python SVN #68532 to fix issue1088 |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
481 def _read_chunked(self, amt): |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
482 chunk_left = self.chunk_left |
30686
8352c42a0a0d
keepalive: don't concatenate strings when reading chunked transfer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30487
diff
changeset
|
483 parts = [] |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
484 |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
485 while True: |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
486 if chunk_left is None: |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
487 line = self.fp.readline() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
488 i = line.find(b';') |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
489 if i >= 0: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
490 line = line[:i] # strip chunk-extensions |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
491 try: |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
492 chunk_left = int(line, 16) |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
493 except ValueError: |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17251
diff
changeset
|
494 # close the connection as protocol synchronization is |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
495 # probably lost |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
496 self.close() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
497 raise httplib.IncompleteRead(b''.join(parts)) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
498 if chunk_left == 0: |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
499 break |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
500 if amt is None: |
30686
8352c42a0a0d
keepalive: don't concatenate strings when reading chunked transfer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30487
diff
changeset
|
501 parts.append(self._safe_read(chunk_left)) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
502 elif amt < chunk_left: |
30686
8352c42a0a0d
keepalive: don't concatenate strings when reading chunked transfer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30487
diff
changeset
|
503 parts.append(self._safe_read(amt)) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
504 self.chunk_left = chunk_left - amt |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
505 return b''.join(parts) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
506 elif amt == chunk_left: |
30686
8352c42a0a0d
keepalive: don't concatenate strings when reading chunked transfer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30487
diff
changeset
|
507 parts.append(self._safe_read(amt)) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
508 self._safe_read(2) # toss the CRLF at the end of the chunk |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
509 self.chunk_left = None |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
510 return b''.join(parts) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
511 else: |
30686
8352c42a0a0d
keepalive: don't concatenate strings when reading chunked transfer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30487
diff
changeset
|
512 parts.append(self._safe_read(chunk_left)) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
513 amt -= chunk_left |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
514 |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
515 # we read the whole chunk, get another |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
516 self._safe_read(2) # toss the CRLF at the end of the chunk |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
517 chunk_left = None |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
518 |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
519 # read and discard trailer up to the CRLF terminator |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
520 ### note: we shouldn't have any trailers! |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
521 while True: |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
522 line = self.fp.readline() |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
523 if not line: |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
524 # a vanishingly small number of sites EOF without |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
525 # sending the trailer |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
526 break |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
527 if line == b'\r\n': |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
528 break |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
529 |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
530 # we read everything; close the "file" |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
531 self.close() |
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
532 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
533 return b''.join(parts) |
7781
a45206455d85
keepalive: borrow code from newer httplib to patch ValueError (issue1088)
Matt Mackall <mpm@selenic.com>
parents:
7622
diff
changeset
|
534 |
30687
5d06f6b73a57
keepalive: remove limit argument from readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30686
diff
changeset
|
535 def readline(self): |
30688
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
536 # Fast path for a line is already available in read buffer. |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
537 i = self._rbuf.find(b'\n') |
30688
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
538 if i >= 0: |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
539 i += 1 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
540 line = self._rbuf[:i] |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
541 self._rbuf = self._rbuf[i:] |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
542 return line |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
543 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
544 # No newline in local buffer. Read until we find one. |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
545 chunks = [self._rbuf] |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
546 i = -1 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
547 readsize = self._rbufsize |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
548 while True: |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
549 new = self._raw_read(readsize) |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
550 if not new: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
551 break |
30688
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
552 |
40033
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
553 self.receivedbytescount += len(new) |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
554 self._connection.receivedbytescount += len(new) |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
555 try: |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
556 self._handler.parent.receivedbytescount += len(new) |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
557 except AttributeError: |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
558 pass |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
559 |
30688
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
560 chunks.append(new) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
561 i = new.find(b'\n') |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
562 if i >= 0: |
30688
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
563 break |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
564 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
565 # We either have exhausted the stream or have a newline in chunks[-1]. |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
566 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
567 # EOF |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
568 if i == -1: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
569 self._rbuf = b'' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
570 return b''.join(chunks) |
30688
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
571 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
572 i += 1 |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
573 self._rbuf = chunks[-1][i:] |
dc5b594f41e9
keepalive: rewrite readline()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30687
diff
changeset
|
574 chunks[-1] = chunks[-1][:i] |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
575 return b''.join(chunks) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
576 |
19872
681f7b9213a4
check-code: check for spaces around = for named parameters
Mads Kiilerich <madski@unity3d.com>
parents:
17700
diff
changeset
|
577 def readlines(self, sizehint=0): |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
578 total = 0 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
579 list = [] |
14494
1ffeeb91c55d
check-code: flag 0/1 used as constant Boolean expression
Martin Geisler <mg@lazybytes.net>
parents:
10394
diff
changeset
|
580 while True: |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
581 line = self.readline() |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
582 if not line: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
583 break |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
584 list.append(line) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
585 total += len(line) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
586 if sizehint and total >= sizehint: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
587 break |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
588 return list |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
589 |
37297
97eedbd5a56c
keepalive: implement readinto()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37119
diff
changeset
|
590 def readinto(self, dest): |
37587
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
591 if self._raw_readinto is None: |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
592 res = self.read(len(dest)) |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
593 if not res: |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
594 return 0 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
595 dest[0 : len(res)] = res |
37587
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
596 return len(res) |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
597 total = len(dest) |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
598 have = len(self._rbuf) |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
599 if have >= total: |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
600 dest[0:total] = self._rbuf[:total] |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
601 self._rbuf = self._rbuf[total:] |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
602 return total |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
603 mv = memoryview(dest) |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
604 got = self._raw_readinto(mv[have:total]) |
40033
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
605 |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
606 self.receivedbytescount += got |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
607 self._connection.receivedbytescount += got |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
608 try: |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
609 self._handler.receivedbytescount += got |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
610 except AttributeError: |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
611 pass |
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
612 |
37587
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
613 dest[0:have] = self._rbuf |
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
614 got += len(self._rbuf) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
615 self._rbuf = b'' |
37587
192b7ad06932
keepalive: rewrite readinto() to not use read()
Augie Fackler <augie@google.com>
parents:
37297
diff
changeset
|
616 return got |
37297
97eedbd5a56c
keepalive: implement readinto()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37119
diff
changeset
|
617 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
618 |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
619 def safesend(self, str): |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
620 """Send `str' to the server. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
621 |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
622 Shamelessly ripped off from httplib to patch a bad behavior. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
623 """ |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
624 # _broken_pipe_resp is an attribute we set in this function |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
625 # if the socket is closed while we're sending data but |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
626 # the server sent us a response before hanging up. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
627 # In that case, we want to pretend to send the rest of the |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
628 # outgoing data, and then let the user use getresponse() |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
629 # (which we wrap) to get this last response before |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
630 # opening a new socket. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
631 if getattr(self, '_broken_pipe_resp', None) is not None: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
632 return |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
633 |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
634 if self.sock is None: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
635 if self.auto_open: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
636 self.connect() |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
637 else: |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16686
diff
changeset
|
638 raise httplib.NotConnected |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
639 |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
640 # send the data to the server. if we get a broken pipe, then close |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
641 # the socket. we want to reconnect when somebody tries to send again. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
642 # |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
643 # NOTE: we DO propagate the error, though, because we cannot simply |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
644 # ignore the error... the caller will know if they can retry. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
645 if self.debuglevel > 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
646 print(b"send:", repr(str)) |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
647 try: |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
648 blocksize = 8192 |
14958
fd246aefedd3
keepalive: use safehasattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents:
14764
diff
changeset
|
649 read = getattr(str, 'read', None) |
fd246aefedd3
keepalive: use safehasattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents:
14764
diff
changeset
|
650 if read is not None: |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
651 if self.debuglevel > 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
652 print(b"sending a read()able") |
14958
fd246aefedd3
keepalive: use safehasattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents:
14764
diff
changeset
|
653 data = read(blocksize) |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
654 while data: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
655 self.sock.sendall(data) |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
656 self.sentbytescount += len(data) |
14958
fd246aefedd3
keepalive: use safehasattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents:
14764
diff
changeset
|
657 data = read(blocksize) |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
658 else: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
659 self.sock.sendall(str) |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
660 self.sentbytescount += len(str) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
22958
diff
changeset
|
661 except socket.error as v: |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
662 reraise = True |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
663 if v.args[0] == errno.EPIPE: # Broken pipe |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
664 if self._HTTPConnection__state == httplib._CS_REQ_SENT: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
665 self._broken_pipe_resp = None |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
666 self._broken_pipe_resp = self.getresponse() |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
667 reraise = False |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
668 self.close() |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
669 if reraise: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
670 raise |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
671 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
672 |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
673 def wrapgetresponse(cls): |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
674 """Wraps getresponse in cls with a broken-pipe sane version. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
675 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
676 |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
677 def safegetresponse(self): |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
678 # In safesend() we might set the _broken_pipe_resp |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
679 # attribute, in which case the socket has already |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
680 # been closed and we just need to give them the response |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
681 # back. Otherwise, we use the normal response path. |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
682 r = getattr(self, '_broken_pipe_resp', None) |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
683 if r is not None: |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
684 return r |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
685 return cls.getresponse(self) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
686 |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
687 safegetresponse.__doc__ = cls.getresponse.__doc__ |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
688 return safegetresponse |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
689 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
690 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
691 class HTTPConnection(httplib.HTTPConnection): |
40031
f2dffa1359c6
url: have httpsconnection inherit from our custom HTTPConnection
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39816
diff
changeset
|
692 # url.httpsconnection inherits from this. So when adding/removing |
f2dffa1359c6
url: have httpsconnection inherit from our custom HTTPConnection
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39816
diff
changeset
|
693 # attributes, be sure to audit httpsconnection() for unintended |
f2dffa1359c6
url: have httpsconnection inherit from our custom HTTPConnection
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39816
diff
changeset
|
694 # consequences. |
f2dffa1359c6
url: have httpsconnection inherit from our custom HTTPConnection
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39816
diff
changeset
|
695 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
696 # use the modified response class |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
697 response_class = HTTPResponse |
9726
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
698 send = safesend |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
699 getresponse = wrapgetresponse(httplib.HTTPConnection) |
430e59ff3437
keepalive: handle broken pipes gracefully during large POSTs
Augie Fackler <durin42@gmail.com>
parents:
8296
diff
changeset
|
700 |
40032
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
701 def __init__(self, *args, **kwargs): |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
702 httplib.HTTPConnection.__init__(self, *args, **kwargs) |
dc82ad1b7f77
keepalive: track request count and bytes sent
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40031
diff
changeset
|
703 self.sentbytescount = 0 |
40033
5e5b06087ec5
keepalive: track number of bytes received from an HTTP response
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40032
diff
changeset
|
704 self.receivedbytescount = 0 |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
705 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
706 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
707 ######################################################################### |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
708 ##### TEST FUNCTIONS |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
709 ######################################################################### |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
710 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
711 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
712 def continuity(url): |
29341
0d83ad967bf8
cleanup: replace uses of util.(md5|sha1|sha256|sha512) with hashlib.\1
Augie Fackler <raf@durin42.com>
parents:
28883
diff
changeset
|
713 md5 = hashlib.md5 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
714 format = b'%25s: %s' |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
715 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
716 # first fetch the file with the normal http handler |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
717 opener = urlreq.buildopener() |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
718 urlreq.installopener(opener) |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
719 fo = urlreq.urlopen(url) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
720 foo = fo.read() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
721 fo.close() |
22505
232d437af120
keepalive: fix how md5 is used
Mike Hommey <mh@glandium.org>
parents:
19872
diff
changeset
|
722 m = md5(foo) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
723 print(format % (b'normal urllib', node.hex(m.digest()))) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
724 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
725 # now install the keepalive handler and try again |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
726 opener = urlreq.buildopener(HTTPHandler()) |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
727 urlreq.installopener(opener) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
728 |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
729 fo = urlreq.urlopen(url) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
730 foo = fo.read() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
731 fo.close() |
22505
232d437af120
keepalive: fix how md5 is used
Mike Hommey <mh@glandium.org>
parents:
19872
diff
changeset
|
732 m = md5(foo) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
733 print(format % (b'keepalive read', node.hex(m.digest()))) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
734 |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
735 fo = urlreq.urlopen(url) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
736 foo = b'' |
14494
1ffeeb91c55d
check-code: flag 0/1 used as constant Boolean expression
Martin Geisler <mg@lazybytes.net>
parents:
10394
diff
changeset
|
737 while True: |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
738 f = fo.readline() |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
739 if f: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
740 foo = foo + f |
34435
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34427
diff
changeset
|
741 else: |
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34427
diff
changeset
|
742 break |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
743 fo.close() |
22505
232d437af120
keepalive: fix how md5 is used
Mike Hommey <mh@glandium.org>
parents:
19872
diff
changeset
|
744 m = md5(foo) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
745 print(format % (b'keepalive readline', node.hex(m.digest()))) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
746 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
747 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
748 def comp(N, url): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
749 print(b' making %i connections to:\n %s' % (N, url)) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
750 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
751 procutil.stdout.write(b' first using the normal urllib handlers') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
752 # first use normal opener |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
753 opener = urlreq.buildopener() |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
754 urlreq.installopener(opener) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
755 t1 = fetch(N, url) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
756 print(b' TIME: %.3f s' % t1) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
757 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
758 procutil.stdout.write(b' now using the keepalive handler ') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
759 # now install the keepalive handler and try again |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
760 opener = urlreq.buildopener(HTTPHandler()) |
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
761 urlreq.installopener(opener) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
762 t2 = fetch(N, url) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
763 print(b' TIME: %.3f s' % t2) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
764 print(b' improvement factor: %.2f' % (t1 / t2)) |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
765 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
766 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
767 def fetch(N, url, delay=0): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
768 import time |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
769 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
770 lens = [] |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
771 starttime = time.time() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
772 for i in range(N): |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
773 if delay and i > 0: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
774 time.sleep(delay) |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
775 fo = urlreq.urlopen(url) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
776 foo = fo.read() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
777 fo.close() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
778 lens.append(len(foo)) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
779 diff = time.time() - starttime |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
780 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
781 j = 0 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
782 for i in lens[1:]: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
783 j = j + 1 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
784 if not i == lens[0]: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
785 print(b"WARNING: inconsistent length on read %i: %i" % (j, i)) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
786 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
787 return diff |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
788 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
789 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
790 def test_timeout(url): |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
791 global DEBUG |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
792 dbbackup = DEBUG |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
793 |
14764
a7d5816087a9
classes: fix class style problems found by b071cd58af50
Thomas Arendsen Hein <thomas@intevation.de>
parents:
14494
diff
changeset
|
794 class FakeLogger(object): |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
9726
diff
changeset
|
795 def debug(self, msg, *args): |
27616
0765d8423fbc
keepalive: use print function
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27507
diff
changeset
|
796 print(msg % args) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
797 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
798 info = warning = error = debug |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
799 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
800 DEBUG = FakeLogger() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
801 print(b" fetching the file to establish a connection") |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
802 fo = urlreq.urlopen(url) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
803 data1 = fo.read() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
804 fo.close() |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
805 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
806 i = 20 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
807 print(b" waiting %i seconds for the server to close the connection" % i) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
808 while i > 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
809 procutil.stdout.write(b'\r %2i' % i) |
37119
d4a2e0d5d042
procutil: bulk-replace util.std* to point to new module
Yuya Nishihara <yuya@tcha.org>
parents:
36295
diff
changeset
|
810 procutil.stdout.flush() |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
811 time.sleep(1) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
812 i -= 1 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
813 procutil.stderr.write(b'\r') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
814 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
815 print(b" fetching the file a second time") |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28278
diff
changeset
|
816 fo = urlreq.urlopen(url) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
817 data2 = fo.read() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
818 fo.close() |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
819 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
820 if data1 == data2: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
821 print(b' data are identical') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
822 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
823 print(b' ERROR: DATA DIFFER') |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
824 |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
825 DEBUG = dbbackup |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
826 |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
827 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
828 def test(url, N=10): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
829 print(b"performing continuity test (making sure stuff isn't corrupted)") |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
830 continuity(url) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
831 print(b'') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
832 print(b"performing speed comparison") |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
833 comp(N, url) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
834 print(b'') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
835 print(b"performing dropped-connection check") |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
836 test_timeout(url) |
2600
c4325f0a9b91
clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2444
diff
changeset
|
837 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
838 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
839 if __name__ == '__main__': |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
840 import time |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
41441
diff
changeset
|
841 |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
842 try: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
843 N = int(sys.argv[1]) |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
844 url = sys.argv[2] |
16688
cfb6682961b8
cleanup: replace naked excepts with more specific ones
Brodie Rao <brodie@sf.io>
parents:
16687
diff
changeset
|
845 except (IndexError, ValueError): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
846 print(b"%s <integer> <url>" % sys.argv[0]) |
2435
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
847 else: |
ff2bac730b99
http client: support persistent connections.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
848 test(url, N) |