Mercurial > hg
changeset 47320:a43d256c041a
dirstate-v2: Add `hg debugupgraderepo` command support
This command changes changes the file formats used inside an existing
repository to what they would be in a new repository with the current config.
For example:
hg debugupgraderepo --config format.exp-dirstate-v2=1 --run
hg debugupgraderepo --config format.exp-dirstate-v2=0 --run
If a repository has a dirstate in v1 format, the first command would upgrade it
to dirstate-v2. Conversely, if a repository has a dirstate in v2 format, the
second command would downgrade it to v1. (Both may also run some unrelated
upgrades.)
Since `format.exp-dirstate-v2` is currently disabled by default, not specifying
it in `--config` or any configuration file would result in the second command.
Differential Revision: https://phab.mercurial-scm.org/D10769
| author | Simon Sapin <simon.sapin@octobus.net> |
|---|---|
| date | Wed, 19 May 2021 18:35:43 +0200 |
| parents | e985a36c2aa3 |
| children | 62225f9da938 |
| files | mercurial/dirstate.py mercurial/upgrade_utils/actions.py mercurial/upgrade_utils/engine.py tests/test-copies-chain-merge.t tests/test-copies-in-changeset.t tests/test-persistent-nodemap.t tests/test-sidedata.t tests/test-upgrade-repo.t |
| diffstat | 8 files changed, 248 insertions(+), 31 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/dirstate.py Fri May 21 17:12:47 2021 +0200 +++ b/mercurial/dirstate.py Wed May 19 18:35:43 2021 +0200 @@ -1775,6 +1775,12 @@ # for consistent view between _pl() and _read() invocations self._pendingmode = None + self._use_dirstate_tree = self._ui.configbool( + b"experimental", + b"dirstate-tree.in-memory", + False, + ) + def addfile(self, *args, **kwargs): return self._rustmap.addfile(*args, **kwargs) @@ -1903,13 +1909,8 @@ raise st = b'' - use_dirstate_tree = self._ui.configbool( - b"experimental", - b"dirstate-tree.in-memory", - False, - ) self._rustmap, parents = rustmod.DirstateMap.new( - use_dirstate_tree, self._use_dirstate_v2, st + self._use_dirstate_tree, self._use_dirstate_v2, st ) if parents and not self._dirtyparents:
--- a/mercurial/upgrade_utils/actions.py Fri May 21 17:12:47 2021 +0200 +++ b/mercurial/upgrade_utils/actions.py Wed May 19 18:35:43 2021 +0200 @@ -80,7 +80,7 @@ # operation in which this improvement was removed postdowngrademessage = None - # By default for now, we assume every improvement touches all the things + # By default we assume that every improvement touches requirements and all revlogs # Whether this improvement touches filelogs touches_filelogs = True @@ -94,6 +94,9 @@ # Whether this improvement changes repository requirements touches_requirements = True + # Whether this improvement touches the dirstate + touches_dirstate = False + allformatvariant = [] # type: List[Type['formatvariant']] @@ -167,6 +170,27 @@ @registerformatvariant +class dirstatev2(requirementformatvariant): + name = b'dirstate-v2' + _requirement = requirements.DIRSTATE_V2_REQUIREMENT + + default = False + + description = _( + b'version 1 of the dirstate file format requires ' + b'reading and parsing it all at once.' + ) + + upgrademessage = _(b'"hg status" will be faster') + + touches_filelogs = False + touches_manifests = False + touches_changelog = False + touches_requirements = True + touches_dirstate = True + + +@registerformatvariant class dotencode(requirementformatvariant): name = b'dotencode' @@ -644,7 +668,6 @@ self.current_requirements = current_requirements # list of upgrade actions the operation will perform self.upgrade_actions = upgrade_actions - self._upgrade_actions_names = set([a.name for a in upgrade_actions]) self.removed_actions = removed_actions self.revlogs_to_process = revlogs_to_process # requirements which will be added by the operation @@ -667,41 +690,42 @@ ] # delta reuse mode of this upgrade operation + upgrade_actions_names = self.upgrade_actions_names self.delta_reuse_mode = revlog.revlog.DELTAREUSEALWAYS - if b're-delta-all' in self._upgrade_actions_names: + if b're-delta-all' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSENEVER - elif b're-delta-parent' in self._upgrade_actions_names: + elif b're-delta-parent' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSESAMEREVS - elif b're-delta-multibase' in self._upgrade_actions_names: + elif b're-delta-multibase' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSESAMEREVS - elif b're-delta-fulladd' in self._upgrade_actions_names: + elif b're-delta-fulladd' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSEFULLADD # should this operation force re-delta of both parents self.force_re_delta_both_parents = ( - b're-delta-multibase' in self._upgrade_actions_names + b're-delta-multibase' in upgrade_actions_names ) # should this operation create a backup of the store self.backup_store = backup_store - # whether the operation touches different revlogs at all or not - self.touches_filelogs = self._touches_filelogs() - self.touches_manifests = self._touches_manifests() - self.touches_changelog = self._touches_changelog() - # whether the operation touches requirements file or not - self.touches_requirements = self._touches_requirements() - self.touches_store = ( - self.touches_filelogs - or self.touches_manifests - or self.touches_changelog - ) + @property + def upgrade_actions_names(self): + return set([a.name for a in self.upgrade_actions]) + + @property + def requirements_only(self): # does the operation only touches repository requirement - self.requirements_only = ( - self.touches_requirements and not self.touches_store + return ( + self.touches_requirements + and not self.touches_filelogs + and not self.touches_manifests + and not self.touches_changelog + and not self.touches_dirstate ) - def _touches_filelogs(self): + @property + def touches_filelogs(self): for a in self.upgrade_actions: # in optimisations, we re-process the revlogs again if a.type == OPTIMISATION: @@ -713,7 +737,8 @@ return True return False - def _touches_manifests(self): + @property + def touches_manifests(self): for a in self.upgrade_actions: # in optimisations, we re-process the revlogs again if a.type == OPTIMISATION: @@ -725,7 +750,8 @@ return True return False - def _touches_changelog(self): + @property + def touches_changelog(self): for a in self.upgrade_actions: # in optimisations, we re-process the revlogs again if a.type == OPTIMISATION: @@ -737,7 +763,8 @@ return True return False - def _touches_requirements(self): + @property + def touches_requirements(self): for a in self.upgrade_actions: # optimisations are used to re-process revlogs and does not result # in a requirement being added or removed @@ -749,6 +776,18 @@ if a.touches_requirements: return True + @property + def touches_dirstate(self): + for a in self.upgrade_actions: + # revlog optimisations do not affect the dirstate + if a.type == OPTIMISATION: + pass + elif a.touches_dirstate: + return True + for a in self.removed_actions: + if a.touches_dirstate: + return True + return False def _write_labeled(self, l, label): @@ -908,6 +947,7 @@ requirements.REVLOGV2_REQUIREMENT, requirements.CHANGELOGV2_REQUIREMENT, requirements.REVLOGV1_REQUIREMENT, + requirements.DIRSTATE_V2_REQUIREMENT, } for name in compression.compengines: engine = compression.compengines[name] @@ -970,6 +1010,7 @@ requirements.REVLOGV1_REQUIREMENT, requirements.REVLOGV2_REQUIREMENT, requirements.CHANGELOGV2_REQUIREMENT, + requirements.DIRSTATE_V2_REQUIREMENT, } for name in compression.compengines: engine = compression.compengines[name]
--- a/mercurial/upgrade_utils/engine.py Fri May 21 17:12:47 2021 +0200 +++ b/mercurial/upgrade_utils/engine.py Wed May 19 18:35:43 2021 +0200 @@ -30,6 +30,7 @@ nodemap, sidedata as sidedatamod, ) +from . import actions as upgrade_actions def get_sidedata_helpers(srcrepo, dstrepo): @@ -458,6 +459,19 @@ ) ) + if upgrade_actions.dirstatev2 in upgrade_op.upgrade_actions: + ui.status(_(b'upgrading to dirstate-v2 from v1\n')) + upgrade_dirstate(ui, srcrepo, upgrade_op, b'v1', b'v2') + upgrade_op.upgrade_actions.remove(upgrade_actions.dirstatev2) + + if upgrade_actions.dirstatev2 in upgrade_op.removed_actions: + ui.status(_(b'downgrading from dirstate-v2 to v1\n')) + upgrade_dirstate(ui, srcrepo, upgrade_op, b'v2', b'v1') + upgrade_op.removed_actions.remove(upgrade_actions.dirstatev2) + + if not (upgrade_op.upgrade_actions or upgrade_op.removed_actions): + return + if upgrade_op.requirements_only: ui.status(_(b'upgrading repository requirements\n')) scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements) @@ -466,7 +480,7 @@ # through the whole cloning process elif ( len(upgrade_op.upgrade_actions) == 1 - and b'persistent-nodemap' in upgrade_op._upgrade_actions_names + and b'persistent-nodemap' in upgrade_op.upgrade_actions_names and not upgrade_op.removed_actions ): ui.status( @@ -591,3 +605,28 @@ backupvfs.unlink(b'store/lock') return backuppath + + +def upgrade_dirstate(ui, srcrepo, upgrade_op, old, new): + if upgrade_op.backup_store: + backuppath = pycompat.mkdtemp( + prefix=b'upgradebackup.', dir=srcrepo.path + ) + ui.status(_(b'replaced files will be backed up at %s\n') % backuppath) + backupvfs = vfsmod.vfs(backuppath) + util.copyfile( + srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires') + ) + util.copyfile( + srcrepo.vfs.join(b'dirstate'), backupvfs.join(b'dirstate') + ) + + assert srcrepo.dirstate._use_dirstate_v2 == (old == b'v2') + srcrepo.dirstate._map._use_dirstate_tree = True + srcrepo.dirstate._map.preload() + srcrepo.dirstate._use_dirstate_v2 = new == b'v2' + srcrepo.dirstate._map._use_dirstate_v2 = srcrepo.dirstate._use_dirstate_v2 + srcrepo.dirstate._dirty = True + srcrepo.dirstate.write(None) + + scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements)
--- a/tests/test-copies-chain-merge.t Fri May 21 17:12:47 2021 +0200 +++ b/tests/test-copies-chain-merge.t Wed May 19 18:35:43 2021 +0200 @@ -1652,6 +1652,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1691,6 +1692,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no
--- a/tests/test-copies-in-changeset.t Fri May 21 17:12:47 2021 +0200 +++ b/tests/test-copies-in-changeset.t Wed May 19 18:35:43 2021 +0200 @@ -35,6 +35,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -52,6 +53,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -426,6 +428,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -456,6 +459,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -483,6 +487,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no
--- a/tests/test-persistent-nodemap.t Fri May 21 17:12:47 2021 +0200 +++ b/tests/test-persistent-nodemap.t Wed May 19 18:35:43 2021 +0200 @@ -57,6 +57,7 @@ $ hg debugformat format-variant repo fncache: yes + dirstate-v2: no dotencode: yes generaldelta: yes share-safe: no @@ -577,6 +578,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -626,6 +628,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no
--- a/tests/test-sidedata.t Fri May 21 17:12:47 2021 +0200 +++ b/tests/test-sidedata.t Wed May 19 18:35:43 2021 +0200 @@ -52,6 +52,7 @@ $ hg debugformat -v -R up-no-side-data format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -68,6 +69,7 @@ $ hg debugformat -v -R up-no-side-data --config experimental.revlogv2=enable-unstable-format-and-corrupt-my-data format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -90,6 +92,7 @@ $ hg debugformat -v -R up-side-data format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -106,6 +109,7 @@ $ hg debugformat -v -R up-side-data --config experimental.revlogv2=no format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no
--- a/tests/test-upgrade-repo.t Fri May 21 17:12:47 2021 +0200 +++ b/tests/test-upgrade-repo.t Wed May 19 18:35:43 2021 +0200 @@ -57,6 +57,7 @@ $ hg debugformat format-variant repo fncache: yes + dirstate-v2: no dotencode: yes generaldelta: yes share-safe: no @@ -72,6 +73,7 @@ $ hg debugformat --verbose format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -88,6 +90,7 @@ $ hg debugformat --verbose --config format.usefncache=no format-variant repo config default fncache: yes no yes + dirstate-v2: no no no dotencode: yes no yes generaldelta: yes yes yes share-safe: no no no @@ -104,6 +107,7 @@ $ hg debugformat --verbose --config format.usefncache=no --color=debug format-variant repo config default [formatvariant.name.mismatchconfig|fncache: ][formatvariant.repo.mismatchconfig| yes][formatvariant.config.special| no][formatvariant.default| yes] + [formatvariant.name.uptodate|dirstate-v2: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] [formatvariant.name.mismatchconfig|dotencode: ][formatvariant.repo.mismatchconfig| yes][formatvariant.config.special| no][formatvariant.default| yes] [formatvariant.name.uptodate|generaldelta: ][formatvariant.repo.uptodate| yes][formatvariant.config.default| yes][formatvariant.default| yes] [formatvariant.name.uptodate|share-safe: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] @@ -126,6 +130,12 @@ "repo": true }, { + "config": false, + "default": false, + "name": "dirstate-v2", + "repo": false + }, + { "config": true, "default": true, "name": "dotencode", @@ -327,6 +337,7 @@ $ hg debugformat format-variant repo fncache: no + dirstate-v2: no dotencode: no generaldelta: no share-safe: no @@ -341,6 +352,7 @@ $ hg debugformat --verbose format-variant repo config default fncache: no yes yes + dirstate-v2: no no no dotencode: no yes yes generaldelta: no yes yes share-safe: no no no @@ -357,6 +369,7 @@ $ hg debugformat --verbose --config format.usegeneraldelta=no format-variant repo config default fncache: no yes yes + dirstate-v2: no no no dotencode: no yes yes generaldelta: no no yes share-safe: no no no @@ -373,6 +386,7 @@ $ hg debugformat --verbose --config format.usegeneraldelta=no --color=debug format-variant repo config default [formatvariant.name.mismatchconfig|fncache: ][formatvariant.repo.mismatchconfig| no][formatvariant.config.default| yes][formatvariant.default| yes] + [formatvariant.name.uptodate|dirstate-v2: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] [formatvariant.name.mismatchconfig|dotencode: ][formatvariant.repo.mismatchconfig| no][formatvariant.config.default| yes][formatvariant.default| yes] [formatvariant.name.mismatchdefault|generaldelta: ][formatvariant.repo.mismatchdefault| no][formatvariant.config.special| no][formatvariant.default| yes] [formatvariant.name.uptodate|share-safe: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] @@ -1355,6 +1369,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1396,6 +1411,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1440,6 +1456,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1490,6 +1507,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1537,6 +1555,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1585,6 +1604,7 @@ $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1613,3 +1633,105 @@ $ hg debugupgraderepo --run nothing to do + +#if rust + +Upgrade to dirstate-v2 + + $ hg debugformat -v --config format.exp-dirstate-v2=1 + format-variant repo config default + fncache: yes yes yes + dirstate-v2: no yes no + dotencode: yes yes yes + generaldelta: yes yes yes + share-safe: no no no + sparserevlog: yes yes yes + persistent-nodemap: yes yes no + copies-sdc: no no no + revlog-v2: yes yes no + changelog-v2: no no no + plain-cl-delta: yes yes yes + compression: zstd zstd zstd + compression-level: default default default + $ hg debugupgraderepo --config format.exp-dirstate-v2=1 --run + upgrade will perform the following actions: + + requirements + preserved: dotencode, exp-revlogv2.2, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, sparserevlog, store + added: exp-dirstate-v2 + + dirstate-v2 + "hg status" will be faster + + processed revlogs: + - all-filelogs + - changelog + - manifest + + beginning upgrade... + repository locked and read-only + creating temporary repository to stage upgraded data: $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + (it is safe to interrupt this process any time before data migration completes) + upgrading to dirstate-v2 from v1 + replaced files will be backed up at $TESTTMP/sparserevlogrepo/.hg/upgradebackup.* (glob) + removing temporary repository $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + $ ls .hg/upgradebackup.*/dirstate + .hg/upgradebackup.*/dirstate (glob) + $ hg debugformat -v + format-variant repo config default + fncache: yes yes yes + dirstate-v2: yes no no + dotencode: yes yes yes + generaldelta: yes yes yes + share-safe: no no no + sparserevlog: yes yes yes + persistent-nodemap: yes yes no + copies-sdc: no no no + revlog-v2: yes yes no + changelog-v2: no no no + plain-cl-delta: yes yes yes + compression: zstd zstd zstd + compression-level: default default default + $ hg status + $ dd status=none bs=12 count=1 if=.hg/dirstate + dirstate-v2 + +Downgrade from dirstate-v2 + + $ hg debugupgraderepo --run + upgrade will perform the following actions: + + requirements + preserved: dotencode, exp-revlogv2.2, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, sparserevlog, store + removed: exp-dirstate-v2 + + processed revlogs: + - all-filelogs + - changelog + - manifest + + beginning upgrade... + repository locked and read-only + creating temporary repository to stage upgraded data: $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + (it is safe to interrupt this process any time before data migration completes) + downgrading from dirstate-v2 to v1 + replaced files will be backed up at $TESTTMP/sparserevlogrepo/.hg/upgradebackup.* (glob) + removing temporary repository $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + $ hg debugformat -v + format-variant repo config default + fncache: yes yes yes + dirstate-v2: no no no + dotencode: yes yes yes + generaldelta: yes yes yes + share-safe: no no no + sparserevlog: yes yes yes + persistent-nodemap: yes yes no + copies-sdc: no no no + revlog-v2: yes yes no + changelog-v2: no no no + plain-cl-delta: yes yes yes + compression: zstd zstd zstd + compression-level: default default default + $ hg status + +#endif