mercurial/keepalive.py

28882:800ec7c048b0

"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.

>>> import urllib2
>>> from keepalive import HTTPHandler
>>> keepalive_handler = HTTPHandler()
>>> opener = urllib2.build_opener(keepalive_handler)
>>> urllib2.install_opener(opener)
>>>
>>> fo = urllib2.urlopen('http://www.python.org')

If a connection to a given host is requested, and all of the existing
connections are still in use, another connection will be opened.  If
the handler tries to use an existing connection but it fails in some
way, it will be closed and removed from the pool.

import errno
import httplib
import socket
import sys
import thread
import urllib2

DEBUG = None

if sys.version_info < (2, 4):
    HANDLE_ERRORS = 1

        return self.do_open(HTTPConnection, req)

    def do_open(self, http_class, req):
        host = req.get_host()
        if not host:
            raise urllib2.URLError('no host given')

        try:
            h = self._cm.get_ready_conn(host)
            while h:
                r = self._reuse_connection(h, req, host)

                               host, id(h))
                self._cm.add(host, h, 0)
                self._start_transaction(h, req)
                r = h.getresponse()
        except (socket.error, httplib.HTTPException) as err:
            raise urllib2.URLError(err)

        # if not a persistent connection, don't try to reuse it
        if r.will_close:
            self._cm.remove(h)

                if 'content-length' not in headers:
                    h.putheader('Content-length', '%d' % len(data))
            else:
                h.putrequest('GET', req.get_selector(), **skipheaders)
        except socket.error as err:
            raise urllib2.URLError(err)
        for k, v in headers.items():
            h.putheader(k, v)
        h.endheaders()
        if req.has_data():
            h.send(data)

class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler):
    pass

class HTTPResponse(httplib.HTTPResponse):
    # we need to subclass HTTPResponse in order to
    # 1) add readline() and readlines() methods

def error_handler(url):
    global HANDLE_ERRORS
    orig = HANDLE_ERRORS
    keepalive_handler = HTTPHandler()
    opener = urllib2.build_opener(keepalive_handler)
    urllib2.install_opener(opener)
    pos = {0: 'off', 1: 'on'}
    for i in (0, 1):
        print("  fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i))
        HANDLE_ERRORS = i
        try:
            fo = urllib2.urlopen(url)
            fo.read()
            fo.close()
            try:
                status, reason = fo.status, fo.reason
            except AttributeError:

    from . import util
    md5 = util.md5
    format = '%25s: %s'

    # first fetch the file with the normal http handler
    opener = urllib2.build_opener()
    urllib2.install_opener(opener)
    fo = urllib2.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % ('normal urllib', m.hexdigest()))

    # now install the keepalive handler and try again
    opener = urllib2.build_opener(HTTPHandler())
    urllib2.install_opener(opener)

    fo = urllib2.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % ('keepalive read', m.hexdigest()))

    fo = urllib2.urlopen(url)
    foo = ''
    while True:
        f = fo.readline()
        if f:
            foo = foo + f

def comp(N, url):
    print('  making %i connections to:\n  %s' % (N, url))

    sys.stdout.write('  first using the normal urllib handlers')
    # first use normal opener
    opener = urllib2.build_opener()
    urllib2.install_opener(opener)
    t1 = fetch(N, url)
    print('  TIME: %.3f s' % t1)

    sys.stdout.write('  now using the keepalive handler       ')
    # now install the keepalive handler and try again
    opener = urllib2.build_opener(HTTPHandler())
    urllib2.install_opener(opener)
    t2 = fetch(N, url)
    print('  TIME: %.3f s' % t2)
    print('  improvement factor: %.2f' % (t1 / t2))

def fetch(N, url, delay=0):

    lens = []
    starttime = time.time()
    for i in range(N):
        if delay and i > 0:
            time.sleep(delay)
        fo = urllib2.urlopen(url)
        foo = fo.read()
        fo.close()
        lens.append(len(foo))
    diff = time.time() - starttime

        def debug(self, msg, *args):
            print(msg % args)
        info = warning = error = debug
    DEBUG = FakeLogger()
    print("  fetching the file to establish a connection")
    fo = urllib2.urlopen(url)
    data1 = fo.read()
    fo.close()

    i = 20
    print("  waiting %i seconds for the server to close the connection" % i)

        time.sleep(1)
        i -= 1
    sys.stderr.write('\r')

    print("  fetching the file a second time")
    fo = urllib2.urlopen(url)
    data2 = fo.read()
    fo.close()

    if data1 == data2:
        print('  data are identical')

28883:032c4c2f802a

"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.

>>> import urllib2
>>> from keepalive import HTTPHandler
>>> keepalive_handler = HTTPHandler()
>>> opener = urlreq.buildopener(keepalive_handler)
>>> urlreq.installopener(opener)
>>>
>>> fo = urlreq.urlopen('http://www.python.org')

If a connection to a given host is requested, and all of the existing
connections are still in use, another connection will be opened.  If
the handler tries to use an existing connection but it fails in some
way, it will be closed and removed from the pool.

import errno
import httplib
import socket
import sys
import thread

from . import (
    util,
)

urlerr = util.urlerr
urlreq = util.urlreq

DEBUG = None

if sys.version_info < (2, 4):
    HANDLE_ERRORS = 1

        return self.do_open(HTTPConnection, req)

    def do_open(self, http_class, req):
        host = req.get_host()
        if not host:
            raise urlerr.urlerror('no host given')

        try:
            h = self._cm.get_ready_conn(host)
            while h:
                r = self._reuse_connection(h, req, host)

                               host, id(h))
                self._cm.add(host, h, 0)
                self._start_transaction(h, req)
                r = h.getresponse()
        except (socket.error, httplib.HTTPException) as err:
            raise urlerr.urlerror(err)

        # if not a persistent connection, don't try to reuse it
        if r.will_close:
            self._cm.remove(h)

                if 'content-length' not in headers:
                    h.putheader('Content-length', '%d' % len(data))
            else:
                h.putrequest('GET', req.get_selector(), **skipheaders)
        except socket.error as err:
            raise urlerr.urlerror(err)
        for k, v in headers.items():
            h.putheader(k, v)
        h.endheaders()
        if req.has_data():
            h.send(data)

class HTTPHandler(KeepAliveHandler, urlreq.httphandler):
    pass

class HTTPResponse(httplib.HTTPResponse):
    # we need to subclass HTTPResponse in order to
    # 1) add readline() and readlines() methods

def error_handler(url):
    global HANDLE_ERRORS
    orig = HANDLE_ERRORS
    keepalive_handler = HTTPHandler()
    opener = urlreq.buildopener(keepalive_handler)
    urlreq.installopener(opener)
    pos = {0: 'off', 1: 'on'}
    for i in (0, 1):
        print("  fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i))
        HANDLE_ERRORS = i
        try:
            fo = urlreq.urlopen(url)
            fo.read()
            fo.close()
            try:
                status, reason = fo.status, fo.reason
            except AttributeError:

    from . import util
    md5 = util.md5
    format = '%25s: %s'

    # first fetch the file with the normal http handler
    opener = urlreq.buildopener()
    urlreq.installopener(opener)
    fo = urlreq.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % ('normal urllib', m.hexdigest()))

    # now install the keepalive handler and try again
    opener = urlreq.buildopener(HTTPHandler())
    urlreq.installopener(opener)

    fo = urlreq.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print(format % ('keepalive read', m.hexdigest()))

    fo = urlreq.urlopen(url)
    foo = ''
    while True:
        f = fo.readline()
        if f:
            foo = foo + f

def comp(N, url):
    print('  making %i connections to:\n  %s' % (N, url))

    sys.stdout.write('  first using the normal urllib handlers')
    # first use normal opener
    opener = urlreq.buildopener()
    urlreq.installopener(opener)
    t1 = fetch(N, url)
    print('  TIME: %.3f s' % t1)

    sys.stdout.write('  now using the keepalive handler       ')
    # now install the keepalive handler and try again
    opener = urlreq.buildopener(HTTPHandler())
    urlreq.installopener(opener)
    t2 = fetch(N, url)
    print('  TIME: %.3f s' % t2)
    print('  improvement factor: %.2f' % (t1 / t2))

def fetch(N, url, delay=0):

    lens = []
    starttime = time.time()
    for i in range(N):
        if delay and i > 0:
            time.sleep(delay)
        fo = urlreq.urlopen(url)
        foo = fo.read()
        fo.close()
        lens.append(len(foo))
    diff = time.time() - starttime

        def debug(self, msg, *args):
            print(msg % args)
        info = warning = error = debug
    DEBUG = FakeLogger()
    print("  fetching the file to establish a connection")
    fo = urlreq.urlopen(url)
    data1 = fo.read()
    fo.close()

    i = 20
    print("  waiting %i seconds for the server to close the connection" % i)

        time.sleep(1)
        i -= 1
    sys.stderr.write('\r')

    print("  fetching the file a second time")
    fo = urlreq.urlopen(url)
    data2 = fo.read()
    fo.close()

    if data1 == data2:
        print('  data are identical')