Mercurial > hg
changeset 48693:de3ac3d2c60b
stream-clone: allow to change persistent-nodemap format during stream clone
Persistent nodemap affect the store format. However it is fairly isolated and
fast to generate locally. So not making it a fixed part of the stream clone is
useful.
This allow clients without persistent-nodemap support (default for client
without Rust enabled, or simply older client). So it make it possible to enable
persistent nodemap on client, where it can provide a massive boost. without too
much consequence.
To do so, we stop using it in the advertisement requirements for streaming and
let the client add/remove the necessary file depending of its configuration.
We still send the files as it seems like a small save to not regenerate them.
In addition, the way we match them will overlap with the changelog-v2/revlog-v2
so we can't simply skip the associated patterns.
Differential Revision: https://phab.mercurial-scm.org/D12096
| author | Pierre-Yves David <pierre-yves.david@octobus.net> |
|---|---|
| date | Thu, 27 Jan 2022 22:24:11 +0100 |
| parents | 4933086bebf5 |
| children | 5c940f9ba3e4 |
| files | mercurial/requirements.py mercurial/revlogutils/nodemap.py mercurial/streamclone.py tests/test-bundle.t tests/test-clone-stream-format.t tests/test-clone-stream.t tests/test-clonebundles.t tests/test-debugcommands.t tests/test-stream-bundle-v2.t |
| diffstat | 9 files changed, 141 insertions(+), 34 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/requirements.py Thu Jan 27 15:22:09 2022 +0100 +++ b/mercurial/requirements.py Thu Jan 27 22:24:11 2022 +0100 @@ -113,7 +113,6 @@ COPIESSDC_REQUIREMENT, GENERALDELTA_REQUIREMENT, INTERNAL_PHASE_REQUIREMENT, - NODEMAP_REQUIREMENT, REVLOG_COMPRESSION_ZSTD, REVLOGV1_REQUIREMENT, REVLOGV2_REQUIREMENT,
--- a/mercurial/revlogutils/nodemap.py Thu Jan 27 15:22:09 2022 +0100 +++ b/mercurial/revlogutils/nodemap.py Thu Jan 27 22:24:11 2022 +0100 @@ -16,6 +16,7 @@ from .. import ( error, + requirements, util, ) from . import docket as docket_mod @@ -34,6 +35,19 @@ pass +def post_stream_cleanup(repo): + """The stream clone might needs to remove some file if persisten nodemap + was dropped while stream cloning + """ + if requirements.REVLOGV1_REQUIREMENT not in repo.requirements: + return + if requirements.NODEMAP_REQUIREMENT in repo.requirements: + return + unfi = repo.unfiltered() + delete_nodemap(None, unfi, unfi.changelog) + delete_nodemap(None, repo, unfi.manifestlog._rootstore._revlog) + + def persisted_data(revlog): """read the nodemap for a revlog from disk""" if revlog._nodemap_file is None:
--- a/mercurial/streamclone.py Thu Jan 27 15:22:09 2022 +0100 +++ b/mercurial/streamclone.py Thu Jan 27 22:24:11 2022 +0100 @@ -27,6 +27,9 @@ store, util, ) +from .revlogutils import ( + nodemap, +) from .utils import ( stringutil, ) @@ -216,6 +219,7 @@ repo.ui, repo.requirements, repo.features ) scmutil.writereporequirements(repo) + nodemap.post_stream_cleanup(repo) if rbranchmap: repo._branchcaches.replace(repo, rbranchmap) @@ -510,6 +514,7 @@ ) consumev1(repo, fp, filecount, bytecount) + nodemap.post_stream_cleanup(repo) class streamcloneapplier(object): @@ -826,6 +831,7 @@ repo.ui, repo.requirements, repo.features ) scmutil.writereporequirements(repo) + nodemap.post_stream_cleanup(repo) def _copy_files(src_vfs_map, dst_vfs_map, entries, progress):
--- a/tests/test-bundle.t Thu Jan 27 15:22:09 2022 +0100 +++ b/tests/test-bundle.t Thu Jan 27 22:24:11 2022 +0100 @@ -297,16 +297,16 @@ $ hg -R test debugcreatestreamclonebundle packed.hg writing 2665 bytes for 6 files - bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog + bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog $ f -B 64 --size --sha1 --hexdump packed.hg - packed.hg: size=2884, sha1=b0c868701f8a9fe44daf094b2f5bf661cf90c789 + packed.hg: size=2865, sha1=353d10311f4befa195d9a1ca4b8e26518115c702 0000: 48 47 53 31 55 4e 00 00 00 00 00 00 00 06 00 00 |HGS1UN..........| - 0010: 00 00 00 00 0a 69 00 4e 67 65 6e 65 72 61 6c 64 |.....i.Ngenerald| - 0020: 65 6c 74 61 2c 70 65 72 73 69 73 74 65 6e 74 2d |elta,persistent-| - 0030: 6e 6f 64 65 6d 61 70 2c 72 65 76 6c 6f 67 2d 63 |nodemap,revlog-c| + 0010: 00 00 00 00 0a 69 00 3b 67 65 6e 65 72 61 6c 64 |.....i.;generald| + 0020: 65 6c 74 61 2c 72 65 76 6c 6f 67 2d 63 6f 6d 70 |elta,revlog-comp| + 0030: 72 65 73 73 69 6f 6e 2d 7a 73 74 64 2c 72 65 76 |ression-zstd,rev| $ hg debugbundle --spec packed.hg - none-packed1;requirements%3Dgeneraldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog + none-packed1;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog #endif #if reporevlogstore no-rust zstd @@ -357,17 +357,17 @@ $ hg -R testnongd debugcreatestreamclonebundle packednongd.hg writing 301 bytes for 3 files - bundle requirements: persistent-nodemap, revlog-compression-zstd, revlogv1 + bundle requirements: revlog-compression-zstd, revlogv1 $ f -B 64 --size --sha1 --hexdump packednongd.hg - packednongd.hg: size=426, sha1=79563ccd6ef779bcfe62a4da64f89a1b308e92e0 + packednongd.hg: size=407, sha1=0b8714422b785ba8eb98c916b41ffd5fb994c9b5 0000: 48 47 53 31 55 4e 00 00 00 00 00 00 00 03 00 00 |HGS1UN..........| - 0010: 00 00 00 00 01 2d 00 34 70 65 72 73 69 73 74 65 |.....-.4persiste| - 0020: 6e 74 2d 6e 6f 64 65 6d 61 70 2c 72 65 76 6c 6f |nt-nodemap,revlo| - 0030: 67 2d 63 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a 73 |g-compression-zs| + 0010: 00 00 00 00 01 2d 00 21 72 65 76 6c 6f 67 2d 63 |.....-.!revlog-c| + 0020: 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a 73 74 64 2c |ompression-zstd,| + 0030: 72 65 76 6c 6f 67 76 31 00 64 61 74 61 2f 66 6f |revlogv1.data/fo| $ hg debugbundle --spec packednongd.hg - none-packed1;requirements%3Dpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1 + none-packed1;requirements%3Drevlog-compression-zstd%2Crevlogv1 #endif @@ -427,7 +427,7 @@ $ hg -R testsecret debugcreatestreamclonebundle packedsecret.hg (warning: stream clone bundle will contain secret revisions) writing 301 bytes for 3 files - bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog + bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog #endif
--- a/tests/test-clone-stream-format.t Thu Jan 27 15:22:09 2022 +0100 +++ b/tests/test-clone-stream-format.t Thu Jan 27 22:24:11 2022 +0100 @@ -4,6 +4,11 @@ #testcases stream-legacy stream-bundle2 + $ cat << EOF >> $HGRCPATH + > [storage] + > revlog.persistent-nodemap.slow-path=allow + > EOF + #if stream-legacy $ cat << EOF >> $HGRCPATH > [server] @@ -13,7 +18,7 @@ Initialize repository - $ hg init server --config format.use-share-safe=yes + $ hg init server --config format.use-share-safe=yes --config format.use-persistent-nodemap=yes $ cd server $ sh $TESTDIR/testlib/stream_clone_setup.sh adding 00changelog-ab349180a0405010.nd @@ -345,3 +350,86 @@ $ killdaemons.py + + +Test streaming from/to repository without a persistent-nodemap +============================================================== + +persistent nodemap affects revlog, but they are easy to generate locally, so we allow it to be changed over a stream clone + + $ rm hg-*.pid errors-*.txt + $ hg clone --pull --config format.use-persistent-nodemap=no server server-no-persistent-nodemap + requesting all changes + adding changesets + adding manifests + adding file changes + added 5004 changesets with 1088 changes to 1088 files (+1 heads) + new changesets 96ee1d7354c4:06ddac466af5 + updating to branch default + 0 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg verify -R server-no-persistent-nodemap + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + checked 5004 changesets with 1088 changes to 1088 files + $ hg -R server serve -p $HGPORT -d --pid-file=hg-1.pid --error errors-1.txt + $ cat hg-1.pid > $DAEMON_PIDS + $ hg -R server-no-persistent-nodemap serve -p $HGPORT2 -d --pid-file=hg-2.pid --error errors-2.txt + $ cat hg-2.pid >> $DAEMON_PIDS + $ hg debugrequires -R server | grep persistent-nodemap + persistent-nodemap + $ hg debugrequires -R server-no-persistent-nodemap | grep persistent-nodemap + [1] + $ ls -1 server/.hg/store/00changelog* + server/.hg/store/00changelog-*.nd (glob) + server/.hg/store/00changelog.d + server/.hg/store/00changelog.i + server/.hg/store/00changelog.n + $ ls -1 server-no-persistent-nodemap/.hg/store/00changelog* + server-no-persistent-nodemap/.hg/store/00changelog.d + server-no-persistent-nodemap/.hg/store/00changelog.i + +persistent-nodemap → no-persistent-nodemap cloning + + $ hg clone --quiet --stream -U http://localhost:$HGPORT clone-remove-persistent-nodemap --config format.use-persistent-nodemap=no + $ cat errors-1.txt + $ hg -R clone-remove-persistent-nodemap verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + checked 5004 changesets with 1088 changes to 1088 files + $ hg debugrequires -R clone-remove-persistent-nodemap | grep persistent-nodemap + [1] + +The persistent-nodemap files should no longer exists + + $ ls -1 clone-remove-persistent-nodemap/.hg/store/00changelog* + clone-remove-persistent-nodemap/.hg/store/00changelog.d + clone-remove-persistent-nodemap/.hg/store/00changelog.i + + +no-persistent-nodemap → persistent-nodemap cloning + + $ hg clone --quiet --stream -U http://localhost:$HGPORT2 clone-add-persistent-nodemap --config format.use-persistent-nodemap=yes + $ cat errors-2.txt + $ hg -R clone-add-persistent-nodemap verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + checked 5004 changesets with 1088 changes to 1088 files + $ hg debugrequires -R clone-add-persistent-nodemap | grep persistent-nodemap + persistent-nodemap + +The persistent-nodemap files should exists + + $ ls -1 clone-add-persistent-nodemap/.hg/store/00changelog* + clone-add-persistent-nodemap/.hg/store/00changelog-*.nd (glob) + clone-add-persistent-nodemap/.hg/store/00changelog.d + clone-add-persistent-nodemap/.hg/store/00changelog.i + clone-add-persistent-nodemap/.hg/store/00changelog.n + + + $ killdaemons.py
--- a/tests/test-clone-stream.t Thu Jan 27 15:22:09 2022 +0100 +++ b/tests/test-clone-stream.t Thu Jan 27 22:24:11 2022 +0100 @@ -338,23 +338,23 @@ #endif #if zstd rust no-dirstate-v2 $ f --size --hex --bytes 256 body - body: size=116331 + body: size=116310 0000: 04 6e 6f 6e 65 48 47 32 30 00 00 00 00 00 00 00 |.noneHG20.......| - 0010: 91 07 53 54 52 45 41 4d 32 00 00 00 00 03 00 09 |..STREAM2.......| - 0020: 06 09 04 0c 55 62 79 74 65 63 6f 75 6e 74 31 30 |....Ubytecount10| + 0010: 7c 07 53 54 52 45 41 4d 32 00 00 00 00 03 00 09 ||.STREAM2.......| + 0020: 06 09 04 0c 40 62 79 74 65 63 6f 75 6e 74 31 30 |....@bytecount10| 0030: 31 32 37 36 66 69 6c 65 63 6f 75 6e 74 31 30 39 |1276filecount109| 0040: 33 72 65 71 75 69 72 65 6d 65 6e 74 73 67 65 6e |3requirementsgen| - 0050: 65 72 61 6c 64 65 6c 74 61 25 32 43 70 65 72 73 |eraldelta%2Cpers| - 0060: 69 73 74 65 6e 74 2d 6e 6f 64 65 6d 61 70 25 32 |istent-nodemap%2| - 0070: 43 72 65 76 6c 6f 67 2d 63 6f 6d 70 72 65 73 73 |Crevlog-compress| - 0080: 69 6f 6e 2d 7a 73 74 64 25 32 43 72 65 76 6c 6f |ion-zstd%2Crevlo| - 0090: 67 76 31 25 32 43 73 70 61 72 73 65 72 65 76 6c |gv1%2Csparserevl| - 00a0: 6f 67 00 00 80 00 73 08 42 64 61 74 61 2f 30 2e |og....s.Bdata/0.| - 00b0: 69 00 03 00 01 00 00 00 00 00 00 00 02 00 00 00 |i...............| - 00c0: 01 00 00 00 00 00 00 00 01 ff ff ff ff ff ff ff |................| - 00d0: ff 80 29 63 a0 49 d3 23 87 bf ce fe 56 67 92 67 |..)c.I.#....Vg.g| - 00e0: 2c 69 d1 ec 39 00 00 00 00 00 00 00 00 00 00 00 |,i..9...........| - 00f0: 00 75 30 73 26 45 64 61 74 61 2f 30 30 63 68 61 |.u0s&Edata/00cha| + 0050: 65 72 61 6c 64 65 6c 74 61 25 32 43 72 65 76 6c |eraldelta%2Crevl| + 0060: 6f 67 2d 63 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a |og-compression-z| + 0070: 73 74 64 25 32 43 72 65 76 6c 6f 67 76 31 25 32 |std%2Crevlogv1%2| + 0080: 43 73 70 61 72 73 65 72 65 76 6c 6f 67 00 00 80 |Csparserevlog...| + 0090: 00 73 08 42 64 61 74 61 2f 30 2e 69 00 03 00 01 |.s.Bdata/0.i....| + 00a0: 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 00 |................| + 00b0: 00 00 00 01 ff ff ff ff ff ff ff ff 80 29 63 a0 |.............)c.| + 00c0: 49 d3 23 87 bf ce fe 56 67 92 67 2c 69 d1 ec 39 |I.#....Vg.g,i..9| + 00d0: 00 00 00 00 00 00 00 00 00 00 00 00 75 30 73 26 |............u0s&| + 00e0: 45 64 61 74 61 2f 30 30 63 68 61 6e 67 65 6c 6f |Edata/00changelo| + 00f0: 67 2d 61 62 33 34 39 31 38 30 61 30 34 30 35 30 |g-ab349180a04050| #endif #if zstd dirstate-v2 $ f --size --hex --bytes 256 body
--- a/tests/test-clonebundles.t Thu Jan 27 15:22:09 2022 +0100 +++ b/tests/test-clonebundles.t Thu Jan 27 22:24:11 2022 +0100 @@ -281,7 +281,7 @@ writing 613 bytes for 4 files bundle requirements: generaldelta, revlogv1, sparserevlog (no-rust no-zstd !) bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog (no-rust zstd !) - bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog (rust !) + bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog (rust !) No bundle spec should work
--- a/tests/test-debugcommands.t Thu Jan 27 15:22:09 2022 +0100 +++ b/tests/test-debugcommands.t Thu Jan 27 22:24:11 2022 +0100 @@ -657,8 +657,8 @@ devel-peer-request: pairs: 81 bytes sending hello command sending between command - remote: 487 - remote: capabilities: batch branchmap $USUAL_BUNDLE2_CAPS$ changegroupsubset getbundle known lookup protocaps pushkey streamreqs=generaldelta,persistent-nodemap,revlog-compression-zstd,revlogv1,sparserevlog unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash + remote: 468 + remote: capabilities: batch branchmap $USUAL_BUNDLE2_CAPS$ changegroupsubset getbundle known lookup protocaps pushkey streamreqs=generaldelta,revlog-compression-zstd,revlogv1,sparserevlog unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash remote: 1 devel-peer-request: protocaps devel-peer-request: caps: * bytes (glob)
--- a/tests/test-stream-bundle-v2.t Thu Jan 27 15:22:09 2022 +0100 +++ b/tests/test-stream-bundle-v2.t Thu Jan 27 22:24:11 2022 +0100 @@ -47,11 +47,11 @@ Stream params: {} stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlogv1%2Csparserevlog} (mandatory: True) (no-zstd !) stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (zstd no-rust !) - stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (rust !) + stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (rust !) $ hg debugbundle --spec bundle.hg none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlogv1%2Csparserevlog (no-zstd !) none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (zstd no-rust !) - none-v2;stream=v2;requirements%3Dgeneraldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (rust !) + none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (rust !) Test that we can apply the bundle as a stream clone bundle