Mercurial > hg

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgext/clonebundles.py	Fri Oct 09 11:22:01 2015 -0700
@@ -0,0 +1,69 @@
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+"""server side extension to advertise pre-generated bundles to seed clones.
+
+The extension essentially serves the content of a .hg/clonebundles.manifest
+file to clients that request it.
+
+The clonebundles.manifest file contains a list of URLs and attributes. URLs
+hold pre-generated bundles that a client fetches and applies. After applying
+the pre-generated bundle, the client will connect back to the original server
+and pull data not in the pre-generated bundle.
+
+Manifest File Format:
+
+The manifest file contains a newline (\n) delimited list of entries.
+
+Each line in this file defines an available bundle. Lines have the format:
+
+    <URL> [<key>=<value]
+
+That is, a URL followed by extra metadata describing it. Metadata keys and
+values should be URL encoded.
+
+This metadata is optional. It is up to server operators to populate this
+metadata.
+
+Keys in UPPERCASE are reserved for use by Mercurial. All non-uppercase keys
+can be used by site installations.
+
+The server operator is responsible for generating the bundle manifest file.
+
+Metadata Attributes:
+
+TBD
+"""
+
+from mercurial import (
+    extensions,
+    wireproto,
+)
+
+testedwith = 'internal'
+
+def capabilities(orig, repo, proto):
+    caps = orig(repo, proto)
+
+    # Only advertise if a manifest exists. This does add some I/O to requests.
+    # But this should be cheaper than a wasted network round trip due to
+    # missing file.
+    if repo.opener.exists('clonebundles.manifest'):
+        caps.append('clonebundles')
+
+    return caps
+
+@wireproto.wireprotocommand('clonebundles', '')
+def bundles(repo, proto):
+    """Server command for returning info for available bundles to seed clones.
+
+    Clients will parse this response and determine what bundle to fetch.
+
+    Other extensions may wrap this command to filter or dynamically emit
+    data depending on the request. e.g. you could advertise URLs for
+    the closest data center given the client's IP address.
+    """
+    return repo.opener.tryread('clonebundles.manifest')
+
+def extsetup(ui):
+    extensions.wrapfunction(wireproto, '_capabilities', capabilities)
--- a/mercurial/exchange.py	Tue Sep 29 16:17:32 2015 -0700
+++ b/mercurial/exchange.py	Fri Oct 09 11:22:01 2015 -0700
@@ -7,12 +7,13 @@

 from i18n import _
 from node import hex, nullid
-import errno, urllib
+import errno, urllib, urllib2
 import util, scmutil, changegroup, base85, error
 import discovery, phases, obsolete, bookmarks as bookmod, bundle2, pushkey
 import lock as lockmod
 import streamclone
 import tags
+import url as urlmod

 def readbundle(ui, fh, fname, vfs=None):
     header = changegroup.readexactly(fh, 4)
@@ -973,6 +974,9 @@
     try:
         pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
         streamclone.maybeperformlegacystreamclone(pullop)
+        # This should ideally be in _pullbundle2(). However, it needs to run
+        # before discovery to avoid extra work.
+        _maybeapplyclonebundle(pullop)
         _pulldiscovery(pullop)
         if pullop.canusebundle2:
             _pullbundle2(pullop)
@@ -1499,3 +1503,88 @@
         if recordout is not None:
             recordout(repo.ui.popbuffer())
     return r
+
+def _maybeapplyclonebundle(pullop):
+    """Apply a clone bundle from a remote, if possible."""
+
+    repo = pullop.repo
+    remote = pullop.remote
+
+    if not repo.ui.configbool('experimental', 'clonebundles', False):
+        return
+
+    if pullop.heads:
+        return
+
+    if not remote.capable('clonebundles'):
+        return
+
+    res = remote._call('clonebundles')
+    entries = parseclonebundlesmanifest(res)
+
+    # TODO filter entries by supported features.
+    # TODO sort entries by user preferences.
+
+    if not entries:
+        repo.ui.note(_('no clone bundles available on remote; '
+                       'falling back to regular clone\n'))
+        return
+
+    url = entries[0]['URL']
+    repo.ui.status(_('applying clone bundle from %s\n') % url)
+    if trypullbundlefromurl(repo.ui, repo, url):
+        repo.ui.status(_('finished applying clone bundle\n'))
+    # Bundle failed.
+    #
+    # We abort by default to avoid the thundering herd of
+    # clients flooding a server that was expecting expensive
+    # clone load to be offloaded.
+    elif repo.ui.configbool('ui', 'clonebundlefallback', False):
+        repo.ui.warn(_('falling back to normal clone\n'))
+    else:
+        raise error.Abort(_('error applying bundle'),
+                          hint=_('consider contacting the server '
+                                 'operator if this error persists'))
+
+def parseclonebundlesmanifest(s):
+    """Parses the raw text of a clone bundles manifest.
+
+    Returns a list of dicts. The dicts have a ``URL`` key corresponding
+    to the URL and other keys are the attributes for the entry.
+    """
+    m = []
+    for line in s.splitlines():
+        fields = line.split()
+        if not fields:
+            continue
+        attrs = {'URL': fields[0]}
+        for rawattr in fields[1:]:
+            key, value = rawattr.split('=', 1)
+            attrs[urllib.unquote(key)] = urllib.unquote(value)
+
+        m.append(attrs)
+
+    return m
+
+def trypullbundlefromurl(ui, repo, url):
+    """Attempt to apply a bundle from a URL."""
+    lock = repo.lock()
+    try:
+        tr = repo.transaction('bundleurl')
+        try:
+            try:
+                fh = urlmod.open(ui, url)
+                cg = readbundle(ui, fh, 'stream')
+                changegroup.addchangegroup(repo, cg, 'clonebundles', url)
+                tr.close()
+                return True
+            except urllib2.HTTPError as e:
+                ui.warn(_('HTTP error fetching bundle: %s\n') % str(e))
+            except urllib2.URLError as e:
+                ui.warn(_('error fetching bundle: %s\n') % e.reason)
+
+            return False
+        finally:
+            tr.release()
+    finally:
+        lock.release()
--- a/mercurial/help/config.txt	Tue Sep 29 16:17:32 2015 -0700
+++ b/mercurial/help/config.txt	Fri Oct 09 11:22:01 2015 -0700
@@ -1412,6 +1412,21 @@
     default ``USER@HOST`` is used instead.
     (default: False)

+``clonebundlefallback``
+    Whether failure to apply an advertised "clone bundle" from a server
+    should result in fallback to a regular clone.
+
+    This is disabled by default because servers advertising "clone
+    bundles" often do so to reduce server load. If advertised bundles
+    start mass failing and clients automatically fall back to a regular
+    clone, this would add significant and unexpected load to the server
+    since the server is expecting clone operations to be offloaded to
+    pre-generated bundles. Failing fast (the default behavior) ensures
+    clients don't overwhelm the server when "clone bundle" application
+    fails.
+
+    (default: False)
+
 ``commitsubrepos``
     Whether to commit modified subrepositories when committing the
     parent repository. If False and one subrepository has uncommitted
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-clonebundles.t	Fri Oct 09 11:22:01 2015 -0700
@@ -0,0 +1,143 @@
+Set up a server
+
+  $ hg init server
+  $ cd server
+  $ cat >> .hg/hgrc << EOF
+  > [extensions]
+  > clonebundles =
+  > EOF
+
+  $ touch foo
+  $ hg -q commit -A -m 'add foo'
+  $ touch bar
+  $ hg -q commit -A -m 'add bar'
+
+  $ hg serve -d -p $HGPORT --pid-file hg.pid --accesslog access.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
+
+Feature disabled by default
+(client should not request manifest)
+
+  $ hg clone -U http://localhost:$HGPORT feature-disabled
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+
+  $ cat server/access.log
+  * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob)
+  * - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob)
+  * - - [*] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=aaff8d2ffbbf07a46dd1f05d8ae7877e3f56e2a2&listkeys=phase%2Cbookmarks (glob)
+  * - - [*] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases (glob)
+
+  $ cat >> $HGRCPATH << EOF
+  > [experimental]
+  > clonebundles = true
+  > EOF
+
+Missing manifest should not result in server lookup
+
+  $ hg --verbose clone -U http://localhost:$HGPORT no-manifest
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+
+  $ tail -4 server/access.log
+  * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob)
+  * - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob)
+  * - - [*] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=aaff8d2ffbbf07a46dd1f05d8ae7877e3f56e2a2&listkeys=phase%2Cbookmarks (glob)
+  * - - [*] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases (glob)
+
+Empty manifest file results in retrieval
+(the extension only checks if the manifest file exists)
+
+  $ touch server/.hg/clonebundles.manifest
+  $ hg --verbose clone -U http://localhost:$HGPORT empty-manifest
+  no clone bundles available on remote; falling back to regular clone
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+
+Manifest file with invalid URL aborts
+
+  $ echo 'http://does.not.exist/bundle.hg' > server/.hg/clonebundles.manifest
+  $ hg clone http://localhost:$HGPORT 404-url
+  applying clone bundle from http://does.not.exist/bundle.hg
+  error fetching bundle: [Errno -2] Name or service not known
+  abort: error applying bundle
+  (consider contacting the server operator if this error persists)
+  [255]
+
+Server is not running aborts
+
+  $ echo "http://localhost:$HGPORT1/bundle.hg" > server/.hg/clonebundles.manifest
+  $ hg clone http://localhost:$HGPORT server-not-runner
+  applying clone bundle from http://localhost:$HGPORT1/bundle.hg
+  error fetching bundle: [Errno 111] Connection refused
+  abort: error applying bundle
+  (consider contacting the server operator if this error persists)
+  [255]
+
+Server returns 404
+
+  $ python $TESTDIR/dumbhttp.py -p $HGPORT1 --pid http.pid
+  $ cat http.pid >> $DAEMON_PIDS
+  $ hg clone http://localhost:$HGPORT running-404
+  applying clone bundle from http://localhost:$HGPORT1/bundle.hg
+  HTTP error fetching bundle: HTTP Error 404: File not found
+  abort: error applying bundle
+  (consider contacting the server operator if this error persists)
+  [255]
+
+We can override failure to fall back to regular clone
+
+  $ hg --config ui.clonebundlefallback=true clone -U http://localhost:$HGPORT 404-fallback
+  applying clone bundle from http://localhost:$HGPORT1/bundle.hg
+  HTTP error fetching bundle: HTTP Error 404: File not found
+  falling back to normal clone
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+
+Bundle with partial content works
+
+  $ hg -R server bundle --type gzip --base null -r 53245c60e682 partial.hg
+  1 changesets found
+
+  $ echo "http://localhost:$HGPORT1/partial.hg" > server/.hg/clonebundles.manifest
+  $ hg clone -U http://localhost:$HGPORT partial-bundle
+  applying clone bundle from http://localhost:$HGPORT1/partial.hg
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  finished applying clone bundle
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+
+Bundle with full content works
+
+  $ hg -R server bundle --type gzip --base null -r tip full.hg
+  2 changesets found
+
+  $ echo "http://localhost:$HGPORT1/full.hg" > server/.hg/clonebundles.manifest
+  $ hg clone -U http://localhost:$HGPORT full-bundle
+  applying clone bundle from http://localhost:$HGPORT1/full.hg
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+  finished applying clone bundle
+  searching for changes
+  no changes found
--- a/tests/test-help.t	Tue Sep 29 16:17:32 2015 -0700
+++ b/tests/test-help.t	Fri Oct 09 11:22:01 2015 -0700
@@ -249,6 +249,8 @@
        bugzilla      hooks for integrating with the Bugzilla bug tracker
        censor        erase file content at a given revision
        churn         command to display statistics about repository history
+       clonebundles  server side extension to advertise pre-generated bundles to
+                     seed clones.
        color         colorize output from some commands
        convert       import revisions from foreign VCS repositories into
                      Mercurial
@@ -1069,6 +1071,8 @@

   Extensions:

+   clonebundles server side extension to advertise pre-generated bundles to seed
+                clones.
    prefixedname matched against word "clone"
    relink       recreates hardlinks between repository clones