Mercurial > hg
comparison hgext/clonebundles.py @ 50431:971dc2369b04
clone-bundles: garbage collect older bundle when generating new ones
See inline documentation for details.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Tue, 24 Mar 2020 03:25:33 +0100 |
parents | 5ae30ff79c76 |
children | 5b70b9f5a2f9 |
comparison
equal
deleted
inserted
replaced
50430:5ae30ff79c76 | 50431:971dc2369b04 |
---|---|
207 | 207 |
208 It is possible to set Mercurial to automatically re-generate clone bundles when | 208 It is possible to set Mercurial to automatically re-generate clone bundles when |
209 new content is available. | 209 new content is available. |
210 | 210 |
211 Mercurial will take care of the process asynchronously. The defined list of | 211 Mercurial will take care of the process asynchronously. The defined list of |
212 bundle type will be generated, uploaded, and advertised. | 212 bundle-type will be generated, uploaded, and advertised. Older bundles will get |
213 decommissioned as newer ones replace them. | |
213 | 214 |
214 Bundles Generation: | 215 Bundles Generation: |
215 ................... | 216 ................... |
216 | 217 |
217 The extension can generate multiple variants of the clone bundle. Each | 218 The extension can generate multiple variants of the clone bundle. Each |
233 | 234 |
234 [clone-bundles] | 235 [clone-bundles] |
235 upload-command=sftp put $HGCB_BUNDLE_PATH \ | 236 upload-command=sftp put $HGCB_BUNDLE_PATH \ |
236 sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME | 237 sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME |
237 | 238 |
239 If the file was already uploaded, the command must still succeed. | |
240 | |
238 After upload, the file should be available at an url defined by | 241 After upload, the file should be available at an url defined by |
239 `clone-bundles.url-template`. | 242 `clone-bundles.url-template`. |
240 | 243 |
241 [clone-bundles] | 244 [clone-bundles] |
242 url-template=https://bundles.host/cache/clone-bundles/{basename} | 245 url-template=https://bundles.host/cache/clone-bundles/{basename} |
246 | |
247 Old bundles cleanup: | |
248 .................... | |
249 | |
250 When new bundles are generated, the older ones are no longer necessary and can | |
251 be removed from storage. This is done through the `clone-bundles.delete-command` | |
252 configuration. The command is given the url of the artifact to delete through | |
253 the `$HGCB_BUNDLE_URL` environment variable. | |
254 | |
255 [clone-bundles] | |
256 delete-command=sftp rm sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME | |
257 | |
258 If the file was already deleted, the command must still succeed. | |
243 """ | 259 """ |
244 | 260 |
245 | 261 |
246 import os | 262 import os |
247 import weakref | 263 import weakref |
296 | 312 |
297 configitem(b'clone-bundles', b'auto-generate.formats', default=list) | 313 configitem(b'clone-bundles', b'auto-generate.formats', default=list) |
298 | 314 |
299 | 315 |
300 configitem(b'clone-bundles', b'upload-command', default=None) | 316 configitem(b'clone-bundles', b'upload-command', default=None) |
317 | |
318 configitem(b'clone-bundles', b'delete-command', default=None) | |
301 | 319 |
302 configitem(b'clone-bundles', b'url-template', default=None) | 320 configitem(b'clone-bundles', b'url-template', default=None) |
303 | 321 |
304 configitem(b'devel', b'debug.clonebundles', default=False) | 322 configitem(b'devel', b'debug.clonebundles', default=False) |
305 | 323 |
664 result = upload_bundle(repo, target) | 682 result = upload_bundle(repo, target) |
665 update_bundle_list(repo, new_bundles=[result]) | 683 update_bundle_list(repo, new_bundles=[result]) |
666 cleanup_tmp_bundle(repo, target) | 684 cleanup_tmp_bundle(repo, target) |
667 | 685 |
668 | 686 |
687 def find_outdated_bundles(repo, bundles): | |
688 """finds outdated bundles""" | |
689 olds = [] | |
690 per_types = {} | |
691 for b in bundles: | |
692 if not b.valid_for(repo): | |
693 olds.append(b) | |
694 continue | |
695 l = per_types.setdefault(b.bundle_type, []) | |
696 l.append(b) | |
697 for key in sorted(per_types): | |
698 all = per_types[key] | |
699 if len(all) > 1: | |
700 all.sort(key=lambda b: b.revs, reverse=True) | |
701 olds.extend(all[1:]) | |
702 return olds | |
703 | |
704 | |
705 def collect_garbage(repo): | |
706 """finds outdated bundles and get them deleted""" | |
707 with repo.clonebundles_lock(): | |
708 bundles = read_auto_gen(repo) | |
709 olds = find_outdated_bundles(repo, bundles) | |
710 for o in olds: | |
711 delete_bundle(repo, o) | |
712 update_bundle_list(repo, del_bundles=olds) | |
713 | |
714 | |
669 def upload_bundle(repo, bundle): | 715 def upload_bundle(repo, bundle): |
670 """upload the result of a GeneratingBundle and return a GeneratedBundle | 716 """upload the result of a GeneratingBundle and return a GeneratedBundle |
671 | 717 |
672 The upload is done using the `clone-bundles.upload-command` | 718 The upload is done using the `clone-bundles.upload-command` |
673 """ | 719 """ |
689 .encode('utf8') | 735 .encode('utf8') |
690 ) | 736 ) |
691 return bundle.uploaded(url, basename) | 737 return bundle.uploaded(url, basename) |
692 | 738 |
693 | 739 |
740 def delete_bundle(repo, bundle): | |
741 """delete a bundle from storage""" | |
742 assert bundle.ready | |
743 msg = b'clone-bundles: deleting bundle %s\n' | |
744 msg %= bundle.basename | |
745 if repo.ui.configbool(b'devel', b'debug.clonebundles'): | |
746 repo.ui.write(msg) | |
747 else: | |
748 repo.ui.debug(msg) | |
749 | |
750 cmd = repo.ui.config(b'clone-bundles', b'delete-command') | |
751 variables = { | |
752 b'HGCB_BUNDLE_URL': bundle.file_url, | |
753 b'HGCB_BASENAME': bundle.basename, | |
754 } | |
755 env = procutil.shellenviron(environ=variables) | |
756 ret = repo.ui.system(cmd, environ=env) | |
757 if ret: | |
758 raise error.Abort(b"command returned status %d: %s" % (ret, cmd)) | |
759 | |
760 | |
694 def auto_bundle_needed_actions(repo, bundles, op_id): | 761 def auto_bundle_needed_actions(repo, bundles, op_id): |
695 """find the list of bundles that need action | 762 """find the list of bundles that need action |
696 | 763 |
697 returns a list of RequestedBundle objects that need to be generated and | 764 returns a list of RequestedBundle objects that need to be generated and |
698 uploaded.""" | 765 uploaded.""" |
699 create_bundles = [] | 766 create_bundles = [] |
767 delete_bundles = [] | |
700 repo = repo.filtered(b"immutable") | 768 repo = repo.filtered(b"immutable") |
701 targets = repo.ui.configlist(b'clone-bundles', b'auto-generate.formats') | 769 targets = repo.ui.configlist(b'clone-bundles', b'auto-generate.formats') |
702 revs = len(repo.changelog) | 770 revs = len(repo.changelog) |
703 generic_data = { | 771 generic_data = { |
704 'revs': revs, | 772 'revs': revs, |
710 for t in targets: | 778 for t in targets: |
711 data = generic_data.copy() | 779 data = generic_data.copy() |
712 data['bundle_type'] = t | 780 data['bundle_type'] = t |
713 b = RequestedBundle(**data) | 781 b = RequestedBundle(**data) |
714 create_bundles.append(b) | 782 create_bundles.append(b) |
715 return create_bundles | 783 delete_bundles.extend(find_outdated_bundles(repo, bundles)) |
784 return create_bundles, delete_bundles | |
716 | 785 |
717 | 786 |
718 def start_one_bundle(repo, bundle): | 787 def start_one_bundle(repo, bundle): |
719 """start the generation of a single bundle file | 788 """start the generation of a single bundle file |
720 | 789 |
757 def debugmakeclonebundles(ui, repo): | 826 def debugmakeclonebundles(ui, repo): |
758 """Internal command to auto-generate debug bundles""" | 827 """Internal command to auto-generate debug bundles""" |
759 requested_bundle = util.pickle.load(procutil.stdin) | 828 requested_bundle = util.pickle.load(procutil.stdin) |
760 procutil.stdin.close() | 829 procutil.stdin.close() |
761 | 830 |
831 collect_garbage(repo) | |
832 | |
762 fname = requested_bundle.suggested_filename | 833 fname = requested_bundle.suggested_filename |
763 fpath = repo.vfs.makedirs(b'tmp-bundles') | 834 fpath = repo.vfs.makedirs(b'tmp-bundles') |
764 fpath = repo.vfs.join(b'tmp-bundles', fname) | 835 fpath = repo.vfs.join(b'tmp-bundles', fname) |
765 bundle = requested_bundle.generating(fpath) | 836 bundle = requested_bundle.generating(fpath) |
766 update_bundle_list(repo, new_bundles=[bundle]) | 837 update_bundle_list(repo, new_bundles=[bundle]) |
776 | 847 |
777 def autobundle(tr): | 848 def autobundle(tr): |
778 repo = reporef() | 849 repo = reporef() |
779 assert repo is not None | 850 assert repo is not None |
780 bundles = read_auto_gen(repo) | 851 bundles = read_auto_gen(repo) |
781 new = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr)) | 852 new, __ = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr)) |
782 for data in new: | 853 for data in new: |
783 start_one_bundle(repo, data) | 854 start_one_bundle(repo, data) |
784 return None | 855 return None |
785 | 856 |
786 return autobundle | 857 return autobundle |