Mercurial > hg
changeset 50431:971dc2369b04
clone-bundles: garbage collect older bundle when generating new ones
See inline documentation for details.
| author | Pierre-Yves David <pierre-yves.david@octobus.net> |
|---|---|
| date | Tue, 24 Mar 2020 03:25:33 +0100 |
| parents | 5ae30ff79c76 |
| children | 5b70b9f5a2f9 |
| files | hgext/clonebundles.py tests/test-clonebundles-autogen.t |
| diffstat | 2 files changed, 100 insertions(+), 3 deletions(-) [+] |
line wrap: on
line diff
--- a/hgext/clonebundles.py Mon Mar 13 17:34:18 2023 +0100 +++ b/hgext/clonebundles.py Tue Mar 24 03:25:33 2020 +0100 @@ -209,7 +209,8 @@ new content is available. Mercurial will take care of the process asynchronously. The defined list of -bundle type will be generated, uploaded, and advertised. +bundle-type will be generated, uploaded, and advertised. Older bundles will get +decommissioned as newer ones replace them. Bundles Generation: ................... @@ -235,11 +236,26 @@ upload-command=sftp put $HGCB_BUNDLE_PATH \ sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME +If the file was already uploaded, the command must still succeed. + After upload, the file should be available at an url defined by `clone-bundles.url-template`. [clone-bundles] url-template=https://bundles.host/cache/clone-bundles/{basename} + +Old bundles cleanup: +.................... + +When new bundles are generated, the older ones are no longer necessary and can +be removed from storage. This is done through the `clone-bundles.delete-command` +configuration. The command is given the url of the artifact to delete through +the `$HGCB_BUNDLE_URL` environment variable. + + [clone-bundles] + delete-command=sftp rm sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME + +If the file was already deleted, the command must still succeed. """ @@ -299,6 +315,8 @@ configitem(b'clone-bundles', b'upload-command', default=None) +configitem(b'clone-bundles', b'delete-command', default=None) + configitem(b'clone-bundles', b'url-template', default=None) configitem(b'devel', b'debug.clonebundles', default=False) @@ -666,6 +684,34 @@ cleanup_tmp_bundle(repo, target) +def find_outdated_bundles(repo, bundles): + """finds outdated bundles""" + olds = [] + per_types = {} + for b in bundles: + if not b.valid_for(repo): + olds.append(b) + continue + l = per_types.setdefault(b.bundle_type, []) + l.append(b) + for key in sorted(per_types): + all = per_types[key] + if len(all) > 1: + all.sort(key=lambda b: b.revs, reverse=True) + olds.extend(all[1:]) + return olds + + +def collect_garbage(repo): + """finds outdated bundles and get them deleted""" + with repo.clonebundles_lock(): + bundles = read_auto_gen(repo) + olds = find_outdated_bundles(repo, bundles) + for o in olds: + delete_bundle(repo, o) + update_bundle_list(repo, del_bundles=olds) + + def upload_bundle(repo, bundle): """upload the result of a GeneratingBundle and return a GeneratedBundle @@ -691,12 +737,34 @@ return bundle.uploaded(url, basename) +def delete_bundle(repo, bundle): + """delete a bundle from storage""" + assert bundle.ready + msg = b'clone-bundles: deleting bundle %s\n' + msg %= bundle.basename + if repo.ui.configbool(b'devel', b'debug.clonebundles'): + repo.ui.write(msg) + else: + repo.ui.debug(msg) + + cmd = repo.ui.config(b'clone-bundles', b'delete-command') + variables = { + b'HGCB_BUNDLE_URL': bundle.file_url, + b'HGCB_BASENAME': bundle.basename, + } + env = procutil.shellenviron(environ=variables) + ret = repo.ui.system(cmd, environ=env) + if ret: + raise error.Abort(b"command returned status %d: %s" % (ret, cmd)) + + def auto_bundle_needed_actions(repo, bundles, op_id): """find the list of bundles that need action returns a list of RequestedBundle objects that need to be generated and uploaded.""" create_bundles = [] + delete_bundles = [] repo = repo.filtered(b"immutable") targets = repo.ui.configlist(b'clone-bundles', b'auto-generate.formats') revs = len(repo.changelog) @@ -712,7 +780,8 @@ data['bundle_type'] = t b = RequestedBundle(**data) create_bundles.append(b) - return create_bundles + delete_bundles.extend(find_outdated_bundles(repo, bundles)) + return create_bundles, delete_bundles def start_one_bundle(repo, bundle): @@ -759,6 +828,8 @@ requested_bundle = util.pickle.load(procutil.stdin) procutil.stdin.close() + collect_garbage(repo) + fname = requested_bundle.suggested_filename fpath = repo.vfs.makedirs(b'tmp-bundles') fpath = repo.vfs.join(b'tmp-bundles', fname) @@ -778,7 +849,7 @@ repo = reporef() assert repo is not None bundles = read_auto_gen(repo) - new = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr)) + new, __ = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr)) for data in new: start_one_bundle(repo, data) return None
--- a/tests/test-clonebundles-autogen.t Mon Mar 13 17:34:18 2023 +0100 +++ b/tests/test-clonebundles-autogen.t Tue Mar 24 03:25:33 2020 +0100 @@ -11,6 +11,7 @@ > [clone-bundles] > auto-generate.formats = v2 > upload-command = cp "\$HGCB_BUNDLE_PATH" "$TESTTMP"/final-upload/ + > delete-command = rm -f "$TESTTMP/final-upload/\$HGCB_BASENAME" > url-template = file://$TESTTMP/final-upload/{basename} > > [devel] @@ -68,3 +69,28 @@ full-v2-2_revs-aaff8d2ffbbf_tip-*_txn.hg (glob) full-v2-4_revs-6427147b985a_tip-*_txn.hg (glob) $ ls -1 ../server/.hg/tmp-bundles + +Older bundles are cleaned up with more pushes +--------------------------------------------- + + $ touch faz + $ hg -q commit -A -m 'add faz' + $ touch fuz + $ hg -q commit -A -m 'add fuz' + $ hg push + pushing to $TESTTMP/server + searching for changes + adding changesets + adding manifests + adding file changes + clone-bundles: deleting bundle full-v2-2_revs-aaff8d2ffbbf_tip-*_txn.hg (glob) + 6 changesets found + added 2 changesets with 2 changes to 2 files + clone-bundles: starting bundle generation: v2 + + $ cat ../server/.hg/clonebundles.manifest + file:/*/$TESTTMP/final-upload/full-v2-6_revs-b1010e95ea00_tip-*_txn.hg BUNDLESPEC=v2 REQUIRESNI=true (glob) + $ ls -1 ../final-upload + full-v2-4_revs-6427147b985a_tip-*_txn.hg (glob) + full-v2-6_revs-b1010e95ea00_tip-*_txn.hg (glob) + $ ls -1 ../server/.hg/tmp-bundles