Mercurial > hg

changeset 50711:40638610c6ee

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
clonebundles: adds a auto-generate.serve-inline option This new option disable the "external" serving of auto generated bundle in favor of using the new "inline bundle" feature that allow serving such bundle through the mercurial protocol. This is a less efficient, but very easy to setup way to start using clone bundle.
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Mon, 19 Jun 2023 18:15:35 +0200
parents 1299525832d0
children 0913a49e020c
files hgext/clonebundles.py tests/test-clonebundles-autogen.t
diffstat 2 files changed, 161 insertions(+), 27 deletions(-) [+]
line wrap: on
line diff
--- a/hgext/clonebundles.py	Mon Jun 19 18:15:27 2023 +0200
+++ b/hgext/clonebundles.py	Mon Jun 19 18:15:35 2023 +0200
@@ -254,9 +254,28 @@
     auto-generate.on-change=yes
     auto-generate.formats= zstd-v2, gzip-v2
 
+Automatic Inline serving
+........................
+
+The simplest way to serve the generated bundle is through the Mercurial
+protocol. However it is not the most efficient as request will still be served
+by that main server. It is useful in case where authentication is complexe or
+when an efficient mirror system is already in use anyway. See the `inline
+clonebundles` section above for details about inline clonebundles
+
+To automatically serve generated bundle through inline clonebundle, simply set
+the following option::
+
+    auto-generate.serve-inline=yes
+
+Enabling this option disable the managed upload and serving explained below.
+
 Bundles Upload and Serving:
 ...........................
 
+This is the most efficient way to serve automatically generated clone bundles,
+but requires some setup.
+
 The generated bundles need to be made available to users through a "public" URL.
 This should be donne through `clone-bundles.upload-command` configuration. The
 value of this command should be a shell command. It will have access to the
@@ -344,6 +363,7 @@
 
 configitem(b'clone-bundles', b'auto-generate.on-change', default=False)
 configitem(b'clone-bundles', b'auto-generate.formats', default=list)
+configitem(b'clone-bundles', b'auto-generate.serve-inline', default=False)
 configitem(b'clone-bundles', b'trigger.below-bundled-ratio', default=0.95)
 configitem(b'clone-bundles', b'trigger.revs', default=1000)
 
@@ -753,45 +773,67 @@
 
     The upload is done using the `clone-bundles.upload-command`
     """
-    cmd = repo.ui.config(b'clone-bundles', b'upload-command')
-    url = repo.ui.config(b'clone-bundles', b'url-template')
+    inline = repo.ui.config(b'clone-bundles', b'auto-generate.serve-inline')
     basename = repo.vfs.basename(bundle.filepath)
-    filepath = procutil.shellquote(bundle.filepath)
-    variables = {
-        b'HGCB_BUNDLE_PATH': filepath,
-        b'HGCB_BUNDLE_BASENAME': basename,
-    }
-    env = procutil.shellenviron(environ=variables)
-    ret = repo.ui.system(cmd, environ=env)
-    if ret:
-        raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
-    url = (
-        url.decode('utf8')
-        .format(basename=basename.decode('utf8'))
-        .encode('utf8')
-    )
-    return bundle.uploaded(url, basename)
+    if inline:
+        dest_dir = repo.vfs.join(bundlecaches.BUNDLE_CACHE_DIR)
+        repo.vfs.makedirs(dest_dir)
+        dest = repo.vfs.join(dest_dir, basename)
+        util.copyfiles(bundle.filepath, dest, hardlink=True)
+        url = bundlecaches.CLONEBUNDLESCHEME + basename
+        return bundle.uploaded(url, basename)
+    else:
+        cmd = repo.ui.config(b'clone-bundles', b'upload-command')
+        url = repo.ui.config(b'clone-bundles', b'url-template')
+        filepath = procutil.shellquote(bundle.filepath)
+        variables = {
+            b'HGCB_BUNDLE_PATH': filepath,
+            b'HGCB_BUNDLE_BASENAME': basename,
+        }
+        env = procutil.shellenviron(environ=variables)
+        ret = repo.ui.system(cmd, environ=env)
+        if ret:
+            raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
+        url = (
+            url.decode('utf8')
+            .format(basename=basename.decode('utf8'))
+            .encode('utf8')
+        )
+        return bundle.uploaded(url, basename)
 
 
 def delete_bundle(repo, bundle):
     """delete a bundle from storage"""
     assert bundle.ready
-    msg = b'clone-bundles: deleting bundle %s\n'
+
+    inline = bundle.file_url.startswith(bundlecaches.CLONEBUNDLESCHEME)
+
+    if inline:
+        msg = b'clone-bundles: deleting inline bundle %s\n'
+    else:
+        msg = b'clone-bundles: deleting bundle %s\n'
     msg %= bundle.basename
     if repo.ui.configbool(b'devel', b'debug.clonebundles'):
         repo.ui.write(msg)
     else:
         repo.ui.debug(msg)
 
-    cmd = repo.ui.config(b'clone-bundles', b'delete-command')
-    variables = {
-        b'HGCB_BUNDLE_URL': bundle.file_url,
-        b'HGCB_BASENAME': bundle.basename,
-    }
-    env = procutil.shellenviron(environ=variables)
-    ret = repo.ui.system(cmd, environ=env)
-    if ret:
-        raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
+    if inline:
+        inline_path = repo.vfs.join(
+            bundlecaches.BUNDLE_CACHE_DIR,
+            bundle.basename,
+        )
+        util.tryunlink(inline_path)
+    else:
+        cmd = repo.ui.config(b'clone-bundles', b'delete-command')
+        variables = {
+            b'HGCB_BUNDLE_URL': bundle.file_url,
+            b'HGCB_BASENAME': bundle.basename,
+        }
+        env = procutil.shellenviron(environ=variables)
+        ret = repo.ui.system(cmd, environ=env)
+        if ret:
+            raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
 
 
 def auto_bundle_needed_actions(repo, bundles, op_id):
--- a/tests/test-clonebundles-autogen.t	Mon Jun 19 18:15:27 2023 +0200
+++ b/tests/test-clonebundles-autogen.t	Mon Jun 19 18:15:35 2023 +0200
@@ -382,3 +382,95 @@
   $ cat ../server/.hg/clonebundles.manifest
   https://example.com/final-upload/full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 REQUIRESNI=true (glob)
   https://example.com/final-upload/full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 REQUIRESNI=true (glob)
+
+Test serving them through inline-clone bundle
+=============================================
+
+  $ cat >> ../server/.hg/hgrc << EOF
+  > [clone-bundles]
+  > auto-generate.serve-inline=yes
+  > EOF
+  $ hg -R ../server/ admin::clone-bundles-clear
+  clone-bundles: deleting bundle full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  clone-bundles: deleting bundle full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+
+initial generation
+------------------
+
+
+  $ hg -R ../server/ admin::clone-bundles-refresh
+  clone-bundles: starting bundle generation: v1
+  11 changesets found
+  clone-bundles: starting bundle generation: v2
+  11 changesets found
+  $ cat ../server/.hg/clonebundles.manifest
+  peer-bundle-cache://full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
+  peer-bundle-cache://full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
+  $ ls -1 ../server/.hg/bundle-cache
+  full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  $ ls -1 ../final-upload
+
+Regeneration eventually cleanup the old ones
+--------------------------------------------
+
+create more content
+  $ touch voit
+  $ hg -q commit -A -m 'add voit'
+  $ touch ar
+  $ hg -q commit -A -m 'add ar'
+  $ hg push
+  pushing to $TESTTMP/server
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+
+check first regeneration
+
+  $ hg -R ../server/ admin::clone-bundles-refresh
+  clone-bundles: starting bundle generation: v1
+  13 changesets found
+  clone-bundles: starting bundle generation: v2
+  13 changesets found
+  $ cat ../server/.hg/clonebundles.manifest
+  peer-bundle-cache://full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
+  peer-bundle-cache://full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
+  $ ls -1 ../server/.hg/bundle-cache
+  full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
+  full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
+  $ ls -1 ../final-upload
+
+check first regeneration (should cleanup the one before that last)
+
+  $ touch "investi"
+  $ hg -q commit -A -m 'add investi'
+  $ touch "lesgisla"
+  $ hg -q commit -A -m 'add lesgisla'
+  $ hg push
+  pushing to $TESTTMP/server
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+
+  $ hg -R ../server/ admin::clone-bundles-refresh
+  clone-bundles: deleting inline bundle full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  clone-bundles: deleting inline bundle full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
+  clone-bundles: starting bundle generation: v1
+  15 changesets found
+  clone-bundles: starting bundle generation: v2
+  15 changesets found
+  $ cat ../server/.hg/clonebundles.manifest
+  peer-bundle-cache://full-v1-15_revs-17615b3984c2_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
+  peer-bundle-cache://full-v2-15_revs-17615b3984c2_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
+  $ ls -1 ../server/.hg/bundle-cache
+  full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
+  full-v1-15_revs-17615b3984c2_tip-*_acbr.hg (glob)
+  full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
+  full-v2-15_revs-17615b3984c2_tip-*_acbr.hg (glob)
+  $ ls -1 ../final-upload