Mercurial > hg
comparison mercurial/upgrade_utils/actions.py @ 46047:4b89cf08d8dc
upgrade: split definition and management of the actions from the main code
This is a second step to clarify and clean up this code. The code responsible
for definition which action exist, are possible and their compatibility if moved
into a sub module.
This clarify the main code and prepare further cleanup.
Differential Revision: https://phab.mercurial-scm.org/D9477
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Tue, 01 Dec 2020 15:11:06 +0100 |
parents | mercurial/upgrade.py@f105c49e89cd |
children | f4f956342cf1 |
comparison
equal
deleted
inserted
replaced
46046:f105c49e89cd | 46047:4b89cf08d8dc |
---|---|
1 # upgrade.py - functions for in place upgrade of Mercurial repository | |
2 # | |
3 # Copyright (c) 2016-present, Gregory Szorc | |
4 # | |
5 # This software may be used and distributed according to the terms of the | |
6 # GNU General Public License version 2 or any later version. | |
7 | |
8 from __future__ import absolute_import | |
9 | |
10 from ..i18n import _ | |
11 from .. import ( | |
12 localrepo, | |
13 requirements, | |
14 util, | |
15 ) | |
16 | |
17 from ..utils import compression | |
18 | |
19 # list of requirements that request a clone of all revlog if added/removed | |
20 RECLONES_REQUIREMENTS = { | |
21 b'generaldelta', | |
22 requirements.SPARSEREVLOG_REQUIREMENT, | |
23 } | |
24 | |
25 | |
26 def requiredsourcerequirements(repo): | |
27 """Obtain requirements required to be present to upgrade a repo. | |
28 | |
29 An upgrade will not be allowed if the repository doesn't have the | |
30 requirements returned by this function. | |
31 """ | |
32 return { | |
33 # Introduced in Mercurial 0.9.2. | |
34 b'revlogv1', | |
35 # Introduced in Mercurial 0.9.2. | |
36 b'store', | |
37 } | |
38 | |
39 | |
40 def blocksourcerequirements(repo): | |
41 """Obtain requirements that will prevent an upgrade from occurring. | |
42 | |
43 An upgrade cannot be performed if the source repository contains a | |
44 requirements in the returned set. | |
45 """ | |
46 return { | |
47 # The upgrade code does not yet support these experimental features. | |
48 # This is an artificial limitation. | |
49 requirements.TREEMANIFEST_REQUIREMENT, | |
50 # This was a precursor to generaldelta and was never enabled by default. | |
51 # It should (hopefully) not exist in the wild. | |
52 b'parentdelta', | |
53 # Upgrade should operate on the actual store, not the shared link. | |
54 requirements.SHARED_REQUIREMENT, | |
55 } | |
56 | |
57 | |
58 def supportremovedrequirements(repo): | |
59 """Obtain requirements that can be removed during an upgrade. | |
60 | |
61 If an upgrade were to create a repository that dropped a requirement, | |
62 the dropped requirement must appear in the returned set for the upgrade | |
63 to be allowed. | |
64 """ | |
65 supported = { | |
66 requirements.SPARSEREVLOG_REQUIREMENT, | |
67 requirements.SIDEDATA_REQUIREMENT, | |
68 requirements.COPIESSDC_REQUIREMENT, | |
69 requirements.NODEMAP_REQUIREMENT, | |
70 requirements.SHARESAFE_REQUIREMENT, | |
71 } | |
72 for name in compression.compengines: | |
73 engine = compression.compengines[name] | |
74 if engine.available() and engine.revlogheader(): | |
75 supported.add(b'exp-compression-%s' % name) | |
76 if engine.name() == b'zstd': | |
77 supported.add(b'revlog-compression-zstd') | |
78 return supported | |
79 | |
80 | |
81 def supporteddestrequirements(repo): | |
82 """Obtain requirements that upgrade supports in the destination. | |
83 | |
84 If the result of the upgrade would create requirements not in this set, | |
85 the upgrade is disallowed. | |
86 | |
87 Extensions should monkeypatch this to add their custom requirements. | |
88 """ | |
89 supported = { | |
90 b'dotencode', | |
91 b'fncache', | |
92 b'generaldelta', | |
93 b'revlogv1', | |
94 b'store', | |
95 requirements.SPARSEREVLOG_REQUIREMENT, | |
96 requirements.SIDEDATA_REQUIREMENT, | |
97 requirements.COPIESSDC_REQUIREMENT, | |
98 requirements.NODEMAP_REQUIREMENT, | |
99 requirements.SHARESAFE_REQUIREMENT, | |
100 } | |
101 for name in compression.compengines: | |
102 engine = compression.compengines[name] | |
103 if engine.available() and engine.revlogheader(): | |
104 supported.add(b'exp-compression-%s' % name) | |
105 if engine.name() == b'zstd': | |
106 supported.add(b'revlog-compression-zstd') | |
107 return supported | |
108 | |
109 | |
110 def allowednewrequirements(repo): | |
111 """Obtain requirements that can be added to a repository during upgrade. | |
112 | |
113 This is used to disallow proposed requirements from being added when | |
114 they weren't present before. | |
115 | |
116 We use a list of allowed requirement additions instead of a list of known | |
117 bad additions because the whitelist approach is safer and will prevent | |
118 future, unknown requirements from accidentally being added. | |
119 """ | |
120 supported = { | |
121 b'dotencode', | |
122 b'fncache', | |
123 b'generaldelta', | |
124 requirements.SPARSEREVLOG_REQUIREMENT, | |
125 requirements.SIDEDATA_REQUIREMENT, | |
126 requirements.COPIESSDC_REQUIREMENT, | |
127 requirements.NODEMAP_REQUIREMENT, | |
128 requirements.SHARESAFE_REQUIREMENT, | |
129 } | |
130 for name in compression.compengines: | |
131 engine = compression.compengines[name] | |
132 if engine.available() and engine.revlogheader(): | |
133 supported.add(b'exp-compression-%s' % name) | |
134 if engine.name() == b'zstd': | |
135 supported.add(b'revlog-compression-zstd') | |
136 return supported | |
137 | |
138 | |
139 def preservedrequirements(repo): | |
140 return set() | |
141 | |
142 | |
143 DEFICIENCY = b'deficiency' | |
144 OPTIMISATION = b'optimization' | |
145 | |
146 | |
147 class improvement(object): | |
148 """Represents an improvement that can be made as part of an upgrade. | |
149 | |
150 The following attributes are defined on each instance: | |
151 | |
152 name | |
153 Machine-readable string uniquely identifying this improvement. It | |
154 will be mapped to an action later in the upgrade process. | |
155 | |
156 type | |
157 Either ``DEFICIENCY`` or ``OPTIMISATION``. A deficiency is an obvious | |
158 problem. An optimization is an action (sometimes optional) that | |
159 can be taken to further improve the state of the repository. | |
160 | |
161 description | |
162 Message intended for humans explaining the improvement in more detail, | |
163 including the implications of it. For ``DEFICIENCY`` types, should be | |
164 worded in the present tense. For ``OPTIMISATION`` types, should be | |
165 worded in the future tense. | |
166 | |
167 upgrademessage | |
168 Message intended for humans explaining what an upgrade addressing this | |
169 issue will do. Should be worded in the future tense. | |
170 """ | |
171 | |
172 def __init__(self, name, type, description, upgrademessage): | |
173 self.name = name | |
174 self.type = type | |
175 self.description = description | |
176 self.upgrademessage = upgrademessage | |
177 | |
178 def __eq__(self, other): | |
179 if not isinstance(other, improvement): | |
180 # This is what python tell use to do | |
181 return NotImplemented | |
182 return self.name == other.name | |
183 | |
184 def __ne__(self, other): | |
185 return not (self == other) | |
186 | |
187 def __hash__(self): | |
188 return hash(self.name) | |
189 | |
190 | |
191 allformatvariant = [] | |
192 | |
193 | |
194 def registerformatvariant(cls): | |
195 allformatvariant.append(cls) | |
196 return cls | |
197 | |
198 | |
199 class formatvariant(improvement): | |
200 """an improvement subclass dedicated to repository format""" | |
201 | |
202 type = DEFICIENCY | |
203 ### The following attributes should be defined for each class: | |
204 | |
205 # machine-readable string uniquely identifying this improvement. it will be | |
206 # mapped to an action later in the upgrade process. | |
207 name = None | |
208 | |
209 # message intended for humans explaining the improvement in more detail, | |
210 # including the implications of it ``DEFICIENCY`` types, should be worded | |
211 # in the present tense. | |
212 description = None | |
213 | |
214 # message intended for humans explaining what an upgrade addressing this | |
215 # issue will do. should be worded in the future tense. | |
216 upgrademessage = None | |
217 | |
218 # value of current Mercurial default for new repository | |
219 default = None | |
220 | |
221 def __init__(self): | |
222 raise NotImplementedError() | |
223 | |
224 @staticmethod | |
225 def fromrepo(repo): | |
226 """current value of the variant in the repository""" | |
227 raise NotImplementedError() | |
228 | |
229 @staticmethod | |
230 def fromconfig(repo): | |
231 """current value of the variant in the configuration""" | |
232 raise NotImplementedError() | |
233 | |
234 | |
235 class requirementformatvariant(formatvariant): | |
236 """formatvariant based on a 'requirement' name. | |
237 | |
238 Many format variant are controlled by a 'requirement'. We define a small | |
239 subclass to factor the code. | |
240 """ | |
241 | |
242 # the requirement that control this format variant | |
243 _requirement = None | |
244 | |
245 @staticmethod | |
246 def _newreporequirements(ui): | |
247 return localrepo.newreporequirements( | |
248 ui, localrepo.defaultcreateopts(ui) | |
249 ) | |
250 | |
251 @classmethod | |
252 def fromrepo(cls, repo): | |
253 assert cls._requirement is not None | |
254 return cls._requirement in repo.requirements | |
255 | |
256 @classmethod | |
257 def fromconfig(cls, repo): | |
258 assert cls._requirement is not None | |
259 return cls._requirement in cls._newreporequirements(repo.ui) | |
260 | |
261 | |
262 @registerformatvariant | |
263 class fncache(requirementformatvariant): | |
264 name = b'fncache' | |
265 | |
266 _requirement = b'fncache' | |
267 | |
268 default = True | |
269 | |
270 description = _( | |
271 b'long and reserved filenames may not work correctly; ' | |
272 b'repository performance is sub-optimal' | |
273 ) | |
274 | |
275 upgrademessage = _( | |
276 b'repository will be more resilient to storing ' | |
277 b'certain paths and performance of certain ' | |
278 b'operations should be improved' | |
279 ) | |
280 | |
281 | |
282 @registerformatvariant | |
283 class dotencode(requirementformatvariant): | |
284 name = b'dotencode' | |
285 | |
286 _requirement = b'dotencode' | |
287 | |
288 default = True | |
289 | |
290 description = _( | |
291 b'storage of filenames beginning with a period or ' | |
292 b'space may not work correctly' | |
293 ) | |
294 | |
295 upgrademessage = _( | |
296 b'repository will be better able to store files ' | |
297 b'beginning with a space or period' | |
298 ) | |
299 | |
300 | |
301 @registerformatvariant | |
302 class generaldelta(requirementformatvariant): | |
303 name = b'generaldelta' | |
304 | |
305 _requirement = b'generaldelta' | |
306 | |
307 default = True | |
308 | |
309 description = _( | |
310 b'deltas within internal storage are unable to ' | |
311 b'choose optimal revisions; repository is larger and ' | |
312 b'slower than it could be; interaction with other ' | |
313 b'repositories may require extra network and CPU ' | |
314 b'resources, making "hg push" and "hg pull" slower' | |
315 ) | |
316 | |
317 upgrademessage = _( | |
318 b'repository storage will be able to create ' | |
319 b'optimal deltas; new repository data will be ' | |
320 b'smaller and read times should decrease; ' | |
321 b'interacting with other repositories using this ' | |
322 b'storage model should require less network and ' | |
323 b'CPU resources, making "hg push" and "hg pull" ' | |
324 b'faster' | |
325 ) | |
326 | |
327 | |
328 @registerformatvariant | |
329 class sharesafe(requirementformatvariant): | |
330 name = b'exp-sharesafe' | |
331 _requirement = requirements.SHARESAFE_REQUIREMENT | |
332 | |
333 default = False | |
334 | |
335 description = _( | |
336 b'old shared repositories do not share source repository ' | |
337 b'requirements and config. This leads to various problems ' | |
338 b'when the source repository format is upgraded or some new ' | |
339 b'extensions are enabled.' | |
340 ) | |
341 | |
342 upgrademessage = _( | |
343 b'Upgrades a repository to share-safe format so that future ' | |
344 b'shares of this repository share its requirements and configs.' | |
345 ) | |
346 | |
347 | |
348 @registerformatvariant | |
349 class sparserevlog(requirementformatvariant): | |
350 name = b'sparserevlog' | |
351 | |
352 _requirement = requirements.SPARSEREVLOG_REQUIREMENT | |
353 | |
354 default = True | |
355 | |
356 description = _( | |
357 b'in order to limit disk reading and memory usage on older ' | |
358 b'version, the span of a delta chain from its root to its ' | |
359 b'end is limited, whatever the relevant data in this span. ' | |
360 b'This can severly limit Mercurial ability to build good ' | |
361 b'chain of delta resulting is much more storage space being ' | |
362 b'taken and limit reusability of on disk delta during ' | |
363 b'exchange.' | |
364 ) | |
365 | |
366 upgrademessage = _( | |
367 b'Revlog supports delta chain with more unused data ' | |
368 b'between payload. These gaps will be skipped at read ' | |
369 b'time. This allows for better delta chains, making a ' | |
370 b'better compression and faster exchange with server.' | |
371 ) | |
372 | |
373 | |
374 @registerformatvariant | |
375 class sidedata(requirementformatvariant): | |
376 name = b'sidedata' | |
377 | |
378 _requirement = requirements.SIDEDATA_REQUIREMENT | |
379 | |
380 default = False | |
381 | |
382 description = _( | |
383 b'Allows storage of extra data alongside a revision, ' | |
384 b'unlocking various caching options.' | |
385 ) | |
386 | |
387 upgrademessage = _(b'Allows storage of extra data alongside a revision.') | |
388 | |
389 | |
390 @registerformatvariant | |
391 class persistentnodemap(requirementformatvariant): | |
392 name = b'persistent-nodemap' | |
393 | |
394 _requirement = requirements.NODEMAP_REQUIREMENT | |
395 | |
396 default = False | |
397 | |
398 description = _( | |
399 b'persist the node -> rev mapping on disk to speedup lookup' | |
400 ) | |
401 | |
402 upgrademessage = _(b'Speedup revision lookup by node id.') | |
403 | |
404 | |
405 @registerformatvariant | |
406 class copiessdc(requirementformatvariant): | |
407 name = b'copies-sdc' | |
408 | |
409 _requirement = requirements.COPIESSDC_REQUIREMENT | |
410 | |
411 default = False | |
412 | |
413 description = _(b'Stores copies information alongside changesets.') | |
414 | |
415 upgrademessage = _( | |
416 b'Allows to use more efficient algorithm to deal with ' b'copy tracing.' | |
417 ) | |
418 | |
419 | |
420 @registerformatvariant | |
421 class removecldeltachain(formatvariant): | |
422 name = b'plain-cl-delta' | |
423 | |
424 default = True | |
425 | |
426 description = _( | |
427 b'changelog storage is using deltas instead of ' | |
428 b'raw entries; changelog reading and any ' | |
429 b'operation relying on changelog data are slower ' | |
430 b'than they could be' | |
431 ) | |
432 | |
433 upgrademessage = _( | |
434 b'changelog storage will be reformated to ' | |
435 b'store raw entries; changelog reading will be ' | |
436 b'faster; changelog size may be reduced' | |
437 ) | |
438 | |
439 @staticmethod | |
440 def fromrepo(repo): | |
441 # Mercurial 4.0 changed changelogs to not use delta chains. Search for | |
442 # changelogs with deltas. | |
443 cl = repo.changelog | |
444 chainbase = cl.chainbase | |
445 return all(rev == chainbase(rev) for rev in cl) | |
446 | |
447 @staticmethod | |
448 def fromconfig(repo): | |
449 return True | |
450 | |
451 | |
452 @registerformatvariant | |
453 class compressionengine(formatvariant): | |
454 name = b'compression' | |
455 default = b'zlib' | |
456 | |
457 description = _( | |
458 b'Compresion algorithm used to compress data. ' | |
459 b'Some engine are faster than other' | |
460 ) | |
461 | |
462 upgrademessage = _( | |
463 b'revlog content will be recompressed with the new algorithm.' | |
464 ) | |
465 | |
466 @classmethod | |
467 def fromrepo(cls, repo): | |
468 # we allow multiple compression engine requirement to co-exist because | |
469 # strickly speaking, revlog seems to support mixed compression style. | |
470 # | |
471 # The compression used for new entries will be "the last one" | |
472 compression = b'zlib' | |
473 for req in repo.requirements: | |
474 prefix = req.startswith | |
475 if prefix(b'revlog-compression-') or prefix(b'exp-compression-'): | |
476 compression = req.split(b'-', 2)[2] | |
477 return compression | |
478 | |
479 @classmethod | |
480 def fromconfig(cls, repo): | |
481 compengines = repo.ui.configlist(b'format', b'revlog-compression') | |
482 # return the first valid value as the selection code would do | |
483 for comp in compengines: | |
484 if comp in util.compengines: | |
485 return comp | |
486 | |
487 # no valide compression found lets display it all for clarity | |
488 return b','.join(compengines) | |
489 | |
490 | |
491 @registerformatvariant | |
492 class compressionlevel(formatvariant): | |
493 name = b'compression-level' | |
494 default = b'default' | |
495 | |
496 description = _(b'compression level') | |
497 | |
498 upgrademessage = _(b'revlog content will be recompressed') | |
499 | |
500 @classmethod | |
501 def fromrepo(cls, repo): | |
502 comp = compressionengine.fromrepo(repo) | |
503 level = None | |
504 if comp == b'zlib': | |
505 level = repo.ui.configint(b'storage', b'revlog.zlib.level') | |
506 elif comp == b'zstd': | |
507 level = repo.ui.configint(b'storage', b'revlog.zstd.level') | |
508 if level is None: | |
509 return b'default' | |
510 return bytes(level) | |
511 | |
512 @classmethod | |
513 def fromconfig(cls, repo): | |
514 comp = compressionengine.fromconfig(repo) | |
515 level = None | |
516 if comp == b'zlib': | |
517 level = repo.ui.configint(b'storage', b'revlog.zlib.level') | |
518 elif comp == b'zstd': | |
519 level = repo.ui.configint(b'storage', b'revlog.zstd.level') | |
520 if level is None: | |
521 return b'default' | |
522 return bytes(level) | |
523 | |
524 | |
525 def finddeficiencies(repo): | |
526 """returns a list of deficiencies that the repo suffer from""" | |
527 deficiencies = [] | |
528 | |
529 # We could detect lack of revlogv1 and store here, but they were added | |
530 # in 0.9.2 and we don't support upgrading repos without these | |
531 # requirements, so let's not bother. | |
532 | |
533 for fv in allformatvariant: | |
534 if not fv.fromrepo(repo): | |
535 deficiencies.append(fv) | |
536 | |
537 return deficiencies | |
538 | |
539 | |
540 ALL_OPTIMISATIONS = [] | |
541 | |
542 | |
543 def register_optimization(obj): | |
544 ALL_OPTIMISATIONS.append(obj) | |
545 return obj | |
546 | |
547 | |
548 register_optimization( | |
549 improvement( | |
550 name=b're-delta-parent', | |
551 type=OPTIMISATION, | |
552 description=_( | |
553 b'deltas within internal storage will be recalculated to ' | |
554 b'choose an optimal base revision where this was not ' | |
555 b'already done; the size of the repository may shrink and ' | |
556 b'various operations may become faster; the first time ' | |
557 b'this optimization is performed could slow down upgrade ' | |
558 b'execution considerably; subsequent invocations should ' | |
559 b'not run noticeably slower' | |
560 ), | |
561 upgrademessage=_( | |
562 b'deltas within internal storage will choose a new ' | |
563 b'base revision if needed' | |
564 ), | |
565 ) | |
566 ) | |
567 | |
568 register_optimization( | |
569 improvement( | |
570 name=b're-delta-multibase', | |
571 type=OPTIMISATION, | |
572 description=_( | |
573 b'deltas within internal storage will be recalculated ' | |
574 b'against multiple base revision and the smallest ' | |
575 b'difference will be used; the size of the repository may ' | |
576 b'shrink significantly when there are many merges; this ' | |
577 b'optimization will slow down execution in proportion to ' | |
578 b'the number of merges in the repository and the amount ' | |
579 b'of files in the repository; this slow down should not ' | |
580 b'be significant unless there are tens of thousands of ' | |
581 b'files and thousands of merges' | |
582 ), | |
583 upgrademessage=_( | |
584 b'deltas within internal storage will choose an ' | |
585 b'optimal delta by computing deltas against multiple ' | |
586 b'parents; may slow down execution time ' | |
587 b'significantly' | |
588 ), | |
589 ) | |
590 ) | |
591 | |
592 register_optimization( | |
593 improvement( | |
594 name=b're-delta-all', | |
595 type=OPTIMISATION, | |
596 description=_( | |
597 b'deltas within internal storage will always be ' | |
598 b'recalculated without reusing prior deltas; this will ' | |
599 b'likely make execution run several times slower; this ' | |
600 b'optimization is typically not needed' | |
601 ), | |
602 upgrademessage=_( | |
603 b'deltas within internal storage will be fully ' | |
604 b'recomputed; this will likely drastically slow down ' | |
605 b'execution time' | |
606 ), | |
607 ) | |
608 ) | |
609 | |
610 register_optimization( | |
611 improvement( | |
612 name=b're-delta-fulladd', | |
613 type=OPTIMISATION, | |
614 description=_( | |
615 b'every revision will be re-added as if it was new ' | |
616 b'content. It will go through the full storage ' | |
617 b'mechanism giving extensions a chance to process it ' | |
618 b'(eg. lfs). This is similar to "re-delta-all" but even ' | |
619 b'slower since more logic is involved.' | |
620 ), | |
621 upgrademessage=_( | |
622 b'each revision will be added as new content to the ' | |
623 b'internal storage; this will likely drastically slow ' | |
624 b'down execution time, but some extensions might need ' | |
625 b'it' | |
626 ), | |
627 ) | |
628 ) | |
629 | |
630 | |
631 def findoptimizations(repo): | |
632 """Determine optimisation that could be used during upgrade""" | |
633 # These are unconditionally added. There is logic later that figures out | |
634 # which ones to apply. | |
635 return list(ALL_OPTIMISATIONS) | |
636 | |
637 | |
638 def determineactions(repo, deficiencies, sourcereqs, destreqs): | |
639 """Determine upgrade actions that will be performed. | |
640 | |
641 Given a list of improvements as returned by ``finddeficiencies`` and | |
642 ``findoptimizations``, determine the list of upgrade actions that | |
643 will be performed. | |
644 | |
645 The role of this function is to filter improvements if needed, apply | |
646 recommended optimizations from the improvements list that make sense, | |
647 etc. | |
648 | |
649 Returns a list of action names. | |
650 """ | |
651 newactions = [] | |
652 | |
653 for d in deficiencies: | |
654 name = d._requirement | |
655 | |
656 # If the action is a requirement that doesn't show up in the | |
657 # destination requirements, prune the action. | |
658 if name is not None and name not in destreqs: | |
659 continue | |
660 | |
661 newactions.append(d) | |
662 | |
663 # FUTURE consider adding some optimizations here for certain transitions. | |
664 # e.g. adding generaldelta could schedule parent redeltas. | |
665 | |
666 return newactions |