comparison mercurial/exchange.py @ 50666:60f9602b413e

clonebundles: add support for inline (streaming) clonebundles

The idea behind inline clonebundles is to send them through
the ssh or https connection to the Mercurial server.
We've been using this specifically for streaming clonebundles,
although it works for 'regular' clonebundles as well
(but is less relevant, since pullbundles exist).

We've had this enabled for around 9 months for a part
of our users.

A few benefits are:
- no need to secure an external system,
  since everything goes through the same Mercurial server
- easier scaling (in our case: no risk of inconsistencies
  between multiple mercurial-server mirrors and nginx clonebundles hosts)

Remaining topics/questions right now:
- The inline clonebundles don't work for https yet.
  This is because httppeer doesn't seem to support sending client
  capabilities.
  I didn't focus on that as my main goal was to get this working
  for ssh.

50646:c814101560d9

    entries = bundlecaches.sortclonebundleentries(repo.ui, entries)

    url = entries[0][b'URL']
    repo.ui.status(_(b'applying clone bundle from %s\n') % url)
    if trypullbundlefromurl(repo.ui, repo, url):
        repo.ui.status(_(b'finished applying clone bundle\n'))
    # Bundle failed.
    #
    # We abort by default to avoid the thundering herd of
    # clients flooding a server that was expecting expensive

                b'"--config ui.clonebundles=false"'
            ),
        )


def trypullbundlefromurl(ui, repo, url):
    """Attempt to apply a bundle from a URL."""
    with repo.lock(), repo.transaction(b'bundleurl') as tr:
        try:
            fh = urlmod.open(ui, url)
            cg = readbundle(ui, fh, b'stream')

            if isinstance(cg, streamclone.streamcloneapplier):
                cg.apply(repo)
            else:

50666:60f9602b413e

    entries = bundlecaches.sortclonebundleentries(repo.ui, entries)

    url = entries[0][b'URL']
    repo.ui.status(_(b'applying clone bundle from %s\n') % url)
    if trypullbundlefromurl(repo.ui, repo, url, remote):
        repo.ui.status(_(b'finished applying clone bundle\n'))
    # Bundle failed.
    #
    # We abort by default to avoid the thundering herd of
    # clients flooding a server that was expecting expensive

                b'"--config ui.clonebundles=false"'
            ),
        )


def inline_clone_bundle_open(ui, url, peer):
    if not peer:
        raise error.Abort(_(b'no remote repository supplied for %s' % url))
    clonebundleid = url[len(bundlecaches.CLONEBUNDLESCHEME) :]
    peerclonebundle = peer.get_inline_clone_bundle(clonebundleid)
    return util.chunkbuffer(peerclonebundle)


def trypullbundlefromurl(ui, repo, url, peer):
    """Attempt to apply a bundle from a URL."""
    with repo.lock(), repo.transaction(b'bundleurl') as tr:
        try:
            if url.startswith(bundlecaches.CLONEBUNDLESCHEME):
                fh = inline_clone_bundle_open(ui, url, peer)
            else:
                fh = urlmod.open(ui, url)
            cg = readbundle(ui, fh, b'stream')

            if isinstance(cg, streamclone.streamcloneapplier):
                cg.apply(repo)
            else: