view hgext/convert/transport.py @ 30817:2b279126b8f5

revlog: use compression engine APIs for decompression

Now that compression engines declare their header in revlog chunks
and can decompress revlog chunks, we refactor revlog.decompress()
to use them.

Making full use of the property that revlog compressor objects are
reusable, revlog instances now maintain a dict mapping an engine's
revlog header to a compressor object. This is not only a performance
optimization for engines where compressor object reuse can result in
better performance, but it also serves as a cache of header values
so we don't need to perform redundant lookups against the compression
engine manager. (Yes, I measured and the overhead of a function call
versus a dict lookup was observed.)

Replacing the previous inline lookup table with a dict lookup was
measured to make chunk reading ~2.5% slower on changelogs and ~4.5%
slower on manifests. So, the inline lookup table has been mostly
preserved so we don't lose performance. This is unfortunate. But
many decompression operations complete in microseconds, so Python
attribute lookup, dict lookup, and function calls do matter.

The impact of this change on mozilla-unified is as follows:

$ hg perfrevlogchunks -c
! chunk
! wall 1.953663 comb 1.950000 user 1.920000 sys 0.030000 (best of 6)
! wall 1.946000 comb 1.940000 user 1.910000 sys 0.030000 (best of 6)
! chunk batch
! wall 1.791075 comb 1.800000 user 1.760000 sys 0.040000 (best of 6)
! wall 1.785690 comb 1.770000 user 1.750000 sys 0.020000 (best of 6)

$ hg perfrevlogchunks -m
! chunk
! wall 2.587262 comb 2.580000 user 2.550000 sys 0.030000 (best of 4)
! wall 2.616330 comb 2.610000 user 2.560000 sys 0.050000 (best of 4)
! chunk batch
! wall 2.427092 comb 2.420000 user 2.400000 sys 0.020000 (best of 5)
! wall 2.462061 comb 2.460000 user 2.400000 sys 0.060000 (best of 4)

Changelog chunk reading is slightly faster but manifest reading is
slower. What gives?

On this repo, 99.85% of changelog entries are zlib compressed (the 'x'
header). On the manifest, 67.5% are zlib and 32.4% are '\0'. This patch
swapped the test order of 'x' and '\0' so now 'x' is tested first. This
makes changelogs faster since they almost always hit the first branch.
This makes a significant percentage of manifest '\0' chunks slower
because that code path now performs an extra test. Yes, I too can't
believe we're able to measure the impact of an if..elif with simple
string compares. I reckon this code would benefit from being written
in C...

# -*- coding: utf-8 -*-

# Copyright (C) 2007 Daniel Holth <[e-mail]>
# This is a stripped-down version of the original bzr-svn transport.py,
# Copyright (C) 2006 Jelmer Vernooij <[e-mail]>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import

import svn.client
import svn.core
import svn.ra

Pool = svn.core.Pool
SubversionException = svn.core.SubversionException

from mercurial import (
    util,
)

# Some older versions of the Python bindings need to be
# explicitly initialized. But what we want to do probably
# won't work worth a darn against those libraries anyway!
svn.ra.initialize()

svn_config = None

def _create_auth_baton(pool):
    """Create a Subversion authentication baton. """
    import svn.client
    # Give the client context baton a suite of authentication
    # providers.h
    providers = [
        svn.client.get_simple_provider(pool),
        svn.client.get_username_provider(pool),
        svn.client.get_ssl_client_cert_file_provider(pool),
        svn.client.get_ssl_client_cert_pw_file_provider(pool),
        svn.client.get_ssl_server_trust_file_provider(pool),
        ]
    # Platform-dependent authentication methods
    getprovider = getattr(svn.core, 'svn_auth_get_platform_specific_provider',
                          None)
    if getprovider:
        # Available in svn >= 1.6
        for name in ('gnome_keyring', 'keychain', 'kwallet', 'windows'):
            for type in ('simple', 'ssl_client_cert_pw', 'ssl_server_trust'):
                p = getprovider(name, type, pool)
                if p:
                    providers.append(p)
    else:
        if util.safehasattr(svn.client, 'get_windows_simple_provider'):
            providers.append(svn.client.get_windows_simple_provider(pool))

    return svn.core.svn_auth_open(providers, pool)

class NotBranchError(SubversionException):
    pass

class SvnRaTransport(object):
    """
    Open an ra connection to a Subversion repository.
    """
    def __init__(self, url="", ra=None):
        self.pool = Pool()
        self.svn_url = url
        self.username = ''
        self.password = ''

        # Only Subversion 1.4 has reparent()
        if ra is None or not util.safehasattr(svn.ra, 'reparent'):
            self.client = svn.client.create_context(self.pool)
            ab = _create_auth_baton(self.pool)
            if False:
                svn.core.svn_auth_set_parameter(
                    ab, svn.core.SVN_AUTH_PARAM_DEFAULT_USERNAME, self.username)
                svn.core.svn_auth_set_parameter(
                    ab, svn.core.SVN_AUTH_PARAM_DEFAULT_PASSWORD, self.password)
            self.client.auth_baton = ab
            global svn_config
            if svn_config is None:
                svn_config = svn.core.svn_config_get_config(None)
            self.client.config = svn_config
            try:
                self.ra = svn.client.open_ra_session(
                    self.svn_url,
                    self.client, self.pool)
            except SubversionException as xxx_todo_changeme:
                (inst, num) = xxx_todo_changeme.args
                if num in (svn.core.SVN_ERR_RA_ILLEGAL_URL,
                           svn.core.SVN_ERR_RA_LOCAL_REPOS_OPEN_FAILED,
                           svn.core.SVN_ERR_BAD_URL):
                    raise NotBranchError(url)
                raise
        else:
            self.ra = ra
            svn.ra.reparent(self.ra, self.svn_url.encode('utf8'))

    class Reporter(object):
        def __init__(self, reporter_data):
            self._reporter, self._baton = reporter_data

        def set_path(self, path, revnum, start_empty, lock_token, pool=None):
            svn.ra.reporter2_invoke_set_path(self._reporter, self._baton,
                        path, revnum, start_empty, lock_token, pool)

        def delete_path(self, path, pool=None):
            svn.ra.reporter2_invoke_delete_path(self._reporter, self._baton,
                    path, pool)

        def link_path(self, path, url, revision, start_empty, lock_token,
                      pool=None):
            svn.ra.reporter2_invoke_link_path(self._reporter, self._baton,
                    path, url, revision, start_empty, lock_token,
                    pool)

        def finish_report(self, pool=None):
            svn.ra.reporter2_invoke_finish_report(self._reporter,
                    self._baton, pool)

        def abort_report(self, pool=None):
            svn.ra.reporter2_invoke_abort_report(self._reporter,
                    self._baton, pool)

    def do_update(self, revnum, path, *args, **kwargs):
        return self.Reporter(svn.ra.do_update(self.ra, revnum, path,
                                              *args, **kwargs))