comparison mercurial/exchange.py @ 45807:74271829ddc0

clonebundles: move a bundle of clone bundle related code to a new module

In the process on general clone bundle automatically, we need to make some
function available more widely. This is a good opportunity to extract a
significant amount of code from `mercurial.exchange` into a new
`mercurial.bundlecaches`. This make `mercurial.exchange` move under the 3K line
range (hooray…).

The module is called `bundlecaches` because I expect it to be eventually useful
for more than just clone bundle (like pull bunbles).

Differential Revision: https://phab.mercurial-scm.org/D9208

45806:88a47cbf063c

from .node import (
    hex,
    nullid,
    nullrev,
)
from .thirdparty import attr
from . import (
    bookmarks as bookmod,
    bundle2,
    changegroup,
    discovery,
    error,
    exchangev2,
    lock as lockmod,

    phases,
    pushkey,
    pycompat,
    requirements,
    scmutil,
    sslutil,
    streamclone,
    url as urlmod,
    util,
    wireprototypes,
)

urlerr = util.urlerr
urlreq = util.urlreq

_NARROWACL_SECTION = b'narrowacl'

# Maps bundle version human names to changegroup versions.
_bundlespeccgversions = {
    b'v1': b'01',
    b'v2': b'02',
    b'packed1': b's1',
    b'bundle2': b'02',  # legacy
}

# Maps bundle version with content opts to choose which part to bundle
_bundlespeccontentopts = {
    b'v1': {
        b'changegroup': True,
        b'cg.version': b'01',
        b'obsolescence': False,
        b'phases': False,
        b'tagsfnodescache': False,
        b'revbranchcache': False,
    },
    b'v2': {
        b'changegroup': True,
        b'cg.version': b'02',
        b'obsolescence': False,
        b'phases': False,
        b'tagsfnodescache': True,
        b'revbranchcache': True,
    },
    b'packed1': {b'cg.version': b's1'},
}
_bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2']

_bundlespecvariants = {
    b"streamv2": {
        b"changegroup": False,
        b"streamv2": True,
        b"tagsfnodescache": False,
        b"revbranchcache": False,
    }
}

# Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
_bundlespecv1compengines = {b'gzip', b'bzip2', b'none'}


@attr.s
class bundlespec(object):
    compression = attr.ib()
    wirecompression = attr.ib()
    version = attr.ib()
    wireversion = attr.ib()
    params = attr.ib()
    contentopts = attr.ib()


def parsebundlespec(repo, spec, strict=True):
    """Parse a bundle string specification into parts.

    Bundle specifications denote a well-defined bundle/exchange format.
    The content of a given specification should not change over time in
    order to ensure that bundles produced by a newer version of Mercurial are
    readable from an older version.

    The string currently has the form:

       <compression>-<type>[;<parameter0>[;<parameter1>]]

    Where <compression> is one of the supported compression formats
    and <type> is (currently) a version string. A ";" can follow the type and
    all text afterwards is interpreted as URI encoded, ";" delimited key=value
    pairs.

    If ``strict`` is True (the default) <compression> is required. Otherwise,
    it is optional.

    Returns a bundlespec object of (compression, version, parameters).
    Compression will be ``None`` if not in strict mode and a compression isn't
    defined.

    An ``InvalidBundleSpecification`` is raised when the specification is
    not syntactically well formed.

    An ``UnsupportedBundleSpecification`` is raised when the compression or
    bundle type/version is not recognized.

    Note: this function will likely eventually return a more complex data
    structure, including bundle2 part information.
    """

    def parseparams(s):
        if b';' not in s:
            return s, {}

        params = {}
        version, paramstr = s.split(b';', 1)

        for p in paramstr.split(b';'):
            if b'=' not in p:
                raise error.InvalidBundleSpecification(
                    _(
                        b'invalid bundle specification: '
                        b'missing "=" in parameter: %s'
                    )
                    % p
                )

            key, value = p.split(b'=', 1)
            key = urlreq.unquote(key)
            value = urlreq.unquote(value)
            params[key] = value

        return version, params

    if strict and b'-' not in spec:
        raise error.InvalidBundleSpecification(
            _(
                b'invalid bundle specification; '
                b'must be prefixed with compression: %s'
            )
            % spec
        )

    if b'-' in spec:
        compression, version = spec.split(b'-', 1)

        if compression not in util.compengines.supportedbundlenames:
            raise error.UnsupportedBundleSpecification(
                _(b'%s compression is not supported') % compression
            )

        version, params = parseparams(version)

        if version not in _bundlespeccgversions:
            raise error.UnsupportedBundleSpecification(
                _(b'%s is not a recognized bundle version') % version
            )
    else:
        # Value could be just the compression or just the version, in which
        # case some defaults are assumed (but only when not in strict mode).
        assert not strict

        spec, params = parseparams(spec)

        if spec in util.compengines.supportedbundlenames:
            compression = spec
            version = b'v1'
            # Generaldelta repos require v2.
            if b'generaldelta' in repo.requirements:
                version = b'v2'
            # Modern compression engines require v2.
            if compression not in _bundlespecv1compengines:
                version = b'v2'
        elif spec in _bundlespeccgversions:
            if spec == b'packed1':
                compression = b'none'
            else:
                compression = b'bzip2'
            version = spec
        else:
            raise error.UnsupportedBundleSpecification(
                _(b'%s is not a recognized bundle specification') % spec
            )

    # Bundle version 1 only supports a known set of compression engines.
    if version == b'v1' and compression not in _bundlespecv1compengines:
        raise error.UnsupportedBundleSpecification(
            _(b'compression engine %s is not supported on v1 bundles')
            % compression
        )

    # The specification for packed1 can optionally declare the data formats
    # required to apply it. If we see this metadata, compare against what the
    # repo supports and error if the bundle isn't compatible.
    if version == b'packed1' and b'requirements' in params:
        requirements = set(params[b'requirements'].split(b','))
        missingreqs = requirements - repo.supportedformats
        if missingreqs:
            raise error.UnsupportedBundleSpecification(
                _(b'missing support for repository features: %s')
                % b', '.join(sorted(missingreqs))
            )

    # Compute contentopts based on the version
    contentopts = _bundlespeccontentopts.get(version, {}).copy()

    # Process the variants
    if b"stream" in params and params[b"stream"] == b"v2":
        variant = _bundlespecvariants[b"streamv2"]
        contentopts.update(variant)

    engine = util.compengines.forbundlename(compression)
    compression, wirecompression = engine.bundletype()
    wireversion = _bundlespeccgversions[version]

    return bundlespec(
        compression, wirecompression, version, wireversion, params, contentopts
    )


def readbundle(ui, fh, fname, vfs=None):
    header = changegroup.readexactly(fh, 4)

    # If we call the wire protocol command, that's good enough to record the
    # attempt.
    pullop.clonebundleattempted = True

    entries = parseclonebundlesmanifest(repo, res)
    if not entries:
        repo.ui.note(
            _(
                b'no clone bundles available on remote; '
                b'falling back to regular clone\n'
            )
        )
        return

    entries = filterclonebundleentries(
        repo, entries, streamclonerequested=pullop.streamclonerequested
    )

    if not entries:
        # There is a thundering herd concern here. However, if a server

        repo.ui.warn(
            _(b'(you may want to report this to the server operator)\n')
        )
        return

    entries = sortclonebundleentries(repo.ui, entries)

    url = entries[0][b'URL']
    repo.ui.status(_(b'applying clone bundle from %s\n') % url)
    if trypullbundlefromurl(repo.ui, repo, url):
        repo.ui.status(_(b'finished applying clone bundle\n'))

                b'"--config ui.clonebundles=false"'
            ),
        )


def parseclonebundlesmanifest(repo, s):
    """Parses the raw text of a clone bundles manifest.

    Returns a list of dicts. The dicts have a ``URL`` key corresponding
    to the URL and other keys are the attributes for the entry.
    """
    m = []
    for line in s.splitlines():
        fields = line.split()
        if not fields:
            continue
        attrs = {b'URL': fields[0]}
        for rawattr in fields[1:]:
            key, value = rawattr.split(b'=', 1)
            key = urlreq.unquote(key)
            value = urlreq.unquote(value)
            attrs[key] = value

            # Parse BUNDLESPEC into components. This makes client-side
            # preferences easier to specify since you can prefer a single
            # component of the BUNDLESPEC.
            if key == b'BUNDLESPEC':
                try:
                    bundlespec = parsebundlespec(repo, value)
                    attrs[b'COMPRESSION'] = bundlespec.compression
                    attrs[b'VERSION'] = bundlespec.version
                except error.InvalidBundleSpecification:
                    pass
                except error.UnsupportedBundleSpecification:
                    pass

        m.append(attrs)

    return m


def isstreamclonespec(bundlespec):
    # Stream clone v1
    if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1':
        return True

    # Stream clone v2
    if (
        bundlespec.wirecompression == b'UN'
        and bundlespec.wireversion == b'02'
        and bundlespec.contentopts.get(b'streamv2')
    ):
        return True

    return False


def filterclonebundleentries(repo, entries, streamclonerequested=False):
    """Remove incompatible clone bundle manifest entries.

    Accepts a list of entries parsed with ``parseclonebundlesmanifest``
    and returns a new list consisting of only the entries that this client
    should be able to apply.

    There is no guarantee we'll be able to apply all returned entries because
    the metadata we use to filter on may be missing or wrong.
    """
    newentries = []
    for entry in entries:
        spec = entry.get(b'BUNDLESPEC')
        if spec:
            try:
                bundlespec = parsebundlespec(repo, spec, strict=True)

                # If a stream clone was requested, filter out non-streamclone
                # entries.
                if streamclonerequested and not isstreamclonespec(bundlespec):
                    repo.ui.debug(
                        b'filtering %s because not a stream clone\n'
                        % entry[b'URL']
                    )
                    continue

            except error.InvalidBundleSpecification as e:
                repo.ui.debug(stringutil.forcebytestr(e) + b'\n')
                continue
            except error.UnsupportedBundleSpecification as e:
                repo.ui.debug(
                    b'filtering %s because unsupported bundle '
                    b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e))
                )
                continue
        # If we don't have a spec and requested a stream clone, we don't know
        # what the entry is so don't attempt to apply it.
        elif streamclonerequested:
            repo.ui.debug(
                b'filtering %s because cannot determine if a stream '
                b'clone bundle\n' % entry[b'URL']
            )
            continue

        if b'REQUIRESNI' in entry and not sslutil.hassni:
            repo.ui.debug(
                b'filtering %s because SNI not supported\n' % entry[b'URL']
            )
            continue

        if b'REQUIREDRAM' in entry:
            try:
                requiredram = util.sizetoint(entry[b'REQUIREDRAM'])
            except error.ParseError:
                repo.ui.debug(
                    b'filtering %s due to a bad REQUIREDRAM attribute\n'
                    % entry[b'URL']
                )
                continue
            actualram = repo.ui.estimatememory()
            if actualram is not None and actualram * 0.66 < requiredram:
                repo.ui.debug(
                    b'filtering %s as it needs more than 2/3 of system memory\n'
                    % entry[b'URL']
                )
                continue

        newentries.append(entry)

    return newentries


class clonebundleentry(object):
    """Represents an item in a clone bundles manifest.

    This rich class is needed to support sorting since sorted() in Python 3
    doesn't support ``cmp`` and our comparison is complex enough that ``key=``
    won't work.
    """

    def __init__(self, value, prefers):
        self.value = value
        self.prefers = prefers

    def _cmp(self, other):
        for prefkey, prefvalue in self.prefers:
            avalue = self.value.get(prefkey)
            bvalue = other.value.get(prefkey)

            # Special case for b missing attribute and a matches exactly.
            if avalue is not None and bvalue is None and avalue == prefvalue:
                return -1

            # Special case for a missing attribute and b matches exactly.
            if bvalue is not None and avalue is None and bvalue == prefvalue:
                return 1

            # We can't compare unless attribute present on both.
            if avalue is None or bvalue is None:
                continue

            # Same values should fall back to next attribute.
            if avalue == bvalue:
                continue

            # Exact matches come first.
            if avalue == prefvalue:
                return -1
            if bvalue == prefvalue:
                return 1

            # Fall back to next attribute.
            continue

        # If we got here we couldn't sort by attributes and prefers. Fall
        # back to index order.
        return 0

    def __lt__(self, other):
        return self._cmp(other) < 0

    def __gt__(self, other):
        return self._cmp(other) > 0

    def __eq__(self, other):
        return self._cmp(other) == 0

    def __le__(self, other):
        return self._cmp(other) <= 0

    def __ge__(self, other):
        return self._cmp(other) >= 0

    def __ne__(self, other):
        return self._cmp(other) != 0


def sortclonebundleentries(ui, entries):
    prefers = ui.configlist(b'ui', b'clonebundleprefers')
    if not prefers:
        return list(entries)

    def _split(p):
        if b'=' not in p:
            hint = _(b"each comma separated item should be key=value pairs")
            raise error.Abort(
                _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint
            )
        return p.split(b'=', 1)

    prefers = [_split(p) for p in prefers]

    items = sorted(clonebundleentry(v, prefers) for v in entries)
    return [i.value for i in items]


def trypullbundlefromurl(ui, repo, url):
    """Attempt to apply a bundle from a URL."""
    with repo.lock(), repo.transaction(b'bundleurl') as tr:
        try:
            fh = urlmod.open(ui, url)

45807:74271829ddc0

from .node import (
    hex,
    nullid,
    nullrev,
)
from . import (
    bookmarks as bookmod,
    bundle2,
    bundlecaches,
    changegroup,
    discovery,
    error,
    exchangev2,
    lock as lockmod,

    phases,
    pushkey,
    pycompat,
    requirements,
    scmutil,
    streamclone,
    url as urlmod,
    util,
    wireprototypes,
)

urlerr = util.urlerr
urlreq = util.urlreq

_NARROWACL_SECTION = b'narrowacl'


def readbundle(ui, fh, fname, vfs=None):
    header = changegroup.readexactly(fh, 4)

    # If we call the wire protocol command, that's good enough to record the
    # attempt.
    pullop.clonebundleattempted = True

    entries = bundlecaches.parseclonebundlesmanifest(repo, res)
    if not entries:
        repo.ui.note(
            _(
                b'no clone bundles available on remote; '
                b'falling back to regular clone\n'
            )
        )
        return

    entries = bundlecaches.filterclonebundleentries(
        repo, entries, streamclonerequested=pullop.streamclonerequested
    )

    if not entries:
        # There is a thundering herd concern here. However, if a server

        repo.ui.warn(
            _(b'(you may want to report this to the server operator)\n')
        )
        return

    entries = bundlecaches.sortclonebundleentries(repo.ui, entries)

    url = entries[0][b'URL']
    repo.ui.status(_(b'applying clone bundle from %s\n') % url)
    if trypullbundlefromurl(repo.ui, repo, url):
        repo.ui.status(_(b'finished applying clone bundle\n'))

                b'"--config ui.clonebundles=false"'
            ),
        )


def trypullbundlefromurl(ui, repo, url):
    """Attempt to apply a bundle from a URL."""
    with repo.lock(), repo.transaction(b'bundleurl') as tr:
        try:
            fh = urlmod.open(ui, url)