Mercurial > evolve
changeset 4130:a1f6b8211016
pullbundle: add caching logic
We now only generate a bundle once (and store it to disk). If we need it again,
we use it directly from disk.
| author | Pierre-Yves David <pierre-yves.david@octobus.net> |
|---|---|
| date | Sun, 23 Sep 2018 00:08:02 +0200 |
| parents | bc4e62a1cb82 |
| children | 788798a9a159 |
| files | hgext3rd/evolve/stablerange.py hgext3rd/pullbundle.py tests/test-pullbundle.t |
| diffstat | 3 files changed, 223 insertions(+), 46 deletions(-) [+] |
line wrap: on
line diff
--- a/hgext3rd/evolve/stablerange.py Sun Sep 23 23:41:08 2018 +0200 +++ b/hgext3rd/evolve/stablerange.py Sun Sep 23 00:08:02 2018 +0200 @@ -195,7 +195,6 @@ assert standard_start < rangedepth slicepoint = standard_start return slicepoint - class stablerangebasic(abstractstablerange): """a very dummy implementation of stablerange
--- a/hgext3rd/pullbundle.py Sun Sep 23 23:41:08 2018 +0200 +++ b/hgext3rd/pullbundle.py Sun Sep 23 00:08:02 2018 +0200 @@ -123,7 +123,7 @@ subranges = canonicalsubranges(repo, stablerange, rangeid) idx = 0 - slices =[] + slices = [] nodes.reverse() for rangeid in subranges: size = rangelength(rangeid) @@ -158,7 +158,7 @@ cut += cursor if cursor == 1: break - cursor //=2 + cursor //= 2 # 2. optimise, bottom part if skip != cut: @@ -216,10 +216,8 @@ if repo.ui.verbose or source == 'bundle': repo.ui.status(_("%d changesets found\n") % len(nodes)) -def makeonecgpart(newpart, repo, rangeid, outgoing, version, source, - bundlecaps, filematcher, cgversions): - # same as upstream code - +def _makenewstream(newpart, repo, outgoing, version, source, + bundlecaps, filematcher, cgversions): old = changegroup._changegroupinfo try: changegroup._changegroupinfo = _changegroupinfo @@ -229,12 +227,76 @@ finally: changegroup._changegroupinfo = old + nbchanges = len(outgoing.missing) + pversion = None + if cgversions: + pversion = version + return (cgstream, nbchanges, pversion) + +def _makepartfromstream(newpart, repo, cgstream, nbchanges, version): + # same as upstream code + part = newpart('changegroup', data=cgstream) - if cgversions: + if version: part.addparam('version', version) - part.addparam('nbchanges', '%d' % len(outgoing.missing), + part.addparam('nbchanges', '%d' % nbchanges, mandatory=False) if 'treemanifest' in repo.requirements: part.addparam('treemanifest', '1') + +# cache management + +def cachedir(repo): + return repo.cachevfs.join('pullbundles') + +def getcache(repo, bundlename): + cdir = cachedir(repo) + bundlepath = os.path.join(cdir, bundlename) + try: + fd = open(bundlepath, 'rb') + return util.filechunkiter(fd) + except IOError as exc: + if exc.errno != errno.ENOENT: + raise + return None + +def cachewriter(repo, bundlename, stream): + cdir = cachedir(repo) + bundlepath = os.path.join(cdir, bundlename) + try: + os.makedirs(cdir) + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + with util.atomictempfile(bundlepath) as cachefile: + for chunk in stream: + cachefile.write(chunk) + yield chunk + +BUNDLEMASK = "%s-%s-%010iskip-%010isize.hg" + +def makeonecgpart(newpart, repo, rangeid, outgoing, version, source, + bundlecaps, filematcher, cgversions): + bundlename = cachedata = None + if rangeid is not None: + nbchanges = repo.stablerange.rangelength(repo, rangeid) + headnode = nodemod.hex(repo.changelog.node(rangeid[0])) + # XXX do we need to use cgversion in there? + bundlename = BUNDLEMASK % (version, headnode, rangeid[1], nbchanges) + cachedata = getcache(repo, bundlename) + if cachedata is None: + partdata = _makenewstream(newpart, repo, outgoing, version, source, + bundlecaps, filematcher, cgversions) + if bundlename is not None: + cgstream = cachewriter(repo, bundlename, partdata[0]) + partdata = (cgstream,) + partdata[1:] + else: + if repo.ui.verbose or source == 'bundle': + repo.ui.status(_("%d changesets found in caches\n") % nbchanges) + pversion = None + if cgversions: + pversion = version + partdata = (cachedata, nbchanges, pversion) + return _makepartfromstream(newpart, repo, *partdata)
--- a/tests/test-pullbundle.t Sun Sep 23 23:41:08 2018 +0200 +++ b/tests/test-pullbundle.t Sun Sep 23 00:08:02 2018 +0200 @@ -221,6 +221,20 @@ new changesets 1ea73414a91b:f864bc82f6a2 (run 'hg update' to get a working copy) + $ touch oldbundles + $ ls -1 server/.hg/cache/pullbundles > newbundles + $ diff -u oldbundles newbundles + --- oldbundles * (glob) + +++ newbundles * (glob) + @@ -0,0 +1,6 @@ + +02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg + +02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg + +02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg + +02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg + +02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg + +02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg + [1] + pull the other missing entries (multiple heads pulled) ------------------------------------------------------ @@ -626,6 +640,36 @@ new changesets 17185c1c22f1:1dded5aafa0f (run 'hg heads' to see heads, 'hg merge' to merge) + $ mv newbundles oldbundles + $ ls -1 server/.hg/cache/pullbundles > newbundles + $ diff -u oldbundles newbundles + --- oldbundles * (glob) + +++ newbundles * (glob) + @@ -1,6 +1,22 @@ + +02-0f376356904fc8c1c6ceaac27990f2fd79b1f8c1-0000001856skip-0000000008size.hg + +02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg + +02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg + +02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg + 02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg + +02-4a6d0f7d07d060b026d9fc690cd89cd26af96e42-0000001264skip-0000000016size.hg + 02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg + +02-5eaa75df73c454c1afff722301a4c73e897de94d-0000001256skip-0000000008size.hg + 02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg + 02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg + +02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg + +02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg + +02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg + +02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg + +02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001235skip-0000000016size.hg + +02-da87a81c5310760f414a933e6550b7e8e60cf241-0000001792skip-0000000064size.hg + +02-dba2fddbf3c28198659046674a512afd616a1519-0000001472skip-0000000032size.hg + +02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg + +02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg + 02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg + 02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg + +02-fb6c210a224903e81e5a8d2ee099cb0c9526ba8c-0000001512skip-0000000004size.hg + [1] + Same Pullin with a different client ----------------------------------- $ hg -R server log -G -T '{rev}:{node}\n' -r '0+1234+(::1234 and (merge() or branchpoint()))' @@ -639,12 +683,12 @@ $ hg -R client2 pull server -r 1234 --debug --config devel.bundle2.debug=yes --traceback| grep -v 'add changeset' pulling from server query 1; heads - 1024 changesets found - 128 changesets found - 64 changesets found - 16 changesets found - 2 changesets found - 1 changesets found + 1024 changesets found in caches + 128 changesets found in caches + 64 changesets found in caches + 16 changesets found in caches + 2 changesets found in caches + 1 changesets found in caches listing keys for "bookmarks" bundle2-output-bundle: "HG20", 9 parts total bundle2-output: start emission of HG20 stream @@ -829,6 +873,10 @@ new changesets 1ea73414a91b:f864bc82f6a2 (run 'hg update' to get a working copy) + $ mv newbundles oldbundles + $ ls -1 server/.hg/cache/pullbundles > newbundles + $ diff -u oldbundles newbundles + $ hg -R server log -G -T '{rev}:{node}\n' -r '1234+head()+(only(head(), 1234) and (merge() or branchpoint()))' o 2130:0f376356904fc8c1c6ceaac27990f2fd79b1f8c1 : @@ -848,22 +896,22 @@ query 1; heads searching for changes all local heads known remotely - 16 changesets found - 29 changesets found - 256 changesets found - 256 changesets found - 64 changesets found - 8 changesets found - 1 changesets found - 4 changesets found - 8 changesets found - 16 changesets found - 128 changesets found - 64 changesets found - 32 changesets found - 8 changesets found - 4 changesets found - 2 changesets found + 16 changesets found in caches + 29 changesets found in caches + 256 changesets found in caches + 256 changesets found in caches + 64 changesets found in caches + 8 changesets found in caches + 1 changesets found in caches + 4 changesets found in caches + 8 changesets found in caches + 16 changesets found in caches + 128 changesets found in caches + 64 changesets found in caches + 32 changesets found in caches + 8 changesets found in caches + 4 changesets found in caches + 2 changesets found in caches listing keys for "bookmarks" bundle2-output-bundle: "HG20", 19 parts total bundle2-output: start emission of HG20 stream @@ -1232,6 +1280,10 @@ new changesets 17185c1c22f1:1dded5aafa0f (run 'hg heads' to see heads, 'hg merge' to merge) + $ mv newbundles oldbundles + $ ls -1 server/.hg/cache/pullbundles > newbundles + $ diff -u oldbundles newbundles + different pull with a different client -------------------------------------- @@ -1252,9 +1304,9 @@ $ hg -R client3 pull server -r 1789 --debug --config devel.bundle2.debug=yes | grep -v 'add changeset' pulling from server query 1; heads - 1024 changesets found + 1024 changesets found in caches 227 changesets found - 29 changesets found + 29 changesets found in caches 128 changesets found 64 changesets found 32 changesets found @@ -1504,6 +1556,40 @@ new changesets 1ea73414a91b:44e80141ad53 (run 'hg update' to get a working copy) + $ mv newbundles oldbundles + $ ls -1 server/.hg/cache/pullbundles > newbundles + $ diff -u oldbundles newbundles + --- oldbundles * (glob) + +++ newbundles * (glob) + @@ -2,19 +2,26 @@ + 02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg + 02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg + 02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg + +02-44e80141ad530a2aa085e9bd9b5311b57eff72ff-0000001522skip-0000000001size.hg + 02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg + 02-4a6d0f7d07d060b026d9fc690cd89cd26af96e42-0000001264skip-0000000016size.hg + 02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg + +02-59e60b258b18cde1e931cf30ce4ae62b49e37abd-0000001520skip-0000000002size.hg + 02-5eaa75df73c454c1afff722301a4c73e897de94d-0000001256skip-0000000008size.hg + 02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg + +02-7a55a4d5ce324910842c893b56173cf2a847cb9d-0000001472skip-0000000032size.hg + 02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg + +02-a4ab7df9d74053fb819c8a1c6a48ad605cc05f8a-0000001504skip-0000000016size.hg + 02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg + 02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg + +02-c12927fef661d2463043347101b90067c2961333-0000001280skip-0000000128size.hg + 02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg + 02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg + +02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001024skip-0000000227size.hg + 02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001235skip-0000000016size.hg + 02-da87a81c5310760f414a933e6550b7e8e60cf241-0000001792skip-0000000064size.hg + 02-dba2fddbf3c28198659046674a512afd616a1519-0000001472skip-0000000032size.hg + +02-dc714c3a5d080165292ba99b097567d0b95e5756-0000001408skip-0000000064size.hg + 02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg + 02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg + 02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg + [1] + $ hg -R server log -G -T '{rev}:{node}\n' -r '1789+head()+parents(roots(only(head(), 1789)))+(only(head(), 1789) and (merge() or branchpoint()))' o 2130:0f376356904fc8c1c6ceaac27990f2fd79b1f8c1 : @@ -1523,19 +1609,19 @@ 1 changesets found 4 changesets found 8 changesets found - 256 changesets found - 64 changesets found - 8 changesets found - 1 changesets found - 4 changesets found - 8 changesets found - 16 changesets found - 128 changesets found - 64 changesets found - 32 changesets found - 8 changesets found - 4 changesets found - 2 changesets found + 256 changesets found in caches + 64 changesets found in caches + 8 changesets found in caches + 1 changesets found in caches + 4 changesets found in caches + 8 changesets found in caches + 16 changesets found in caches + 128 changesets found in caches + 64 changesets found in caches + 32 changesets found in caches + 8 changesets found in caches + 4 changesets found in caches + 2 changesets found in caches listing keys for "bookmarks" bundle2-output-bundle: "HG20", 19 parts total bundle2-output: start emission of HG20 stream @@ -1901,3 +1987,33 @@ updating the branch cache new changesets c232505f58fd:1dded5aafa0f (run 'hg heads' to see heads, 'hg merge' to merge) + + $ mv newbundles oldbundles + $ ls -1 server/.hg/cache/pullbundles > newbundles + $ diff -u oldbundles newbundles + --- oldbundles * (glob) + +++ newbundles * (glob) + @@ -1,5 +1,6 @@ + 02-0f376356904fc8c1c6ceaac27990f2fd79b1f8c1-0000001856skip-0000000008size.hg + 02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg + +02-1ed78f99f705cb819a02f1227c217728d008e461-0000001524skip-0000000004size.hg + 02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg + 02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg + 02-44e80141ad530a2aa085e9bd9b5311b57eff72ff-0000001522skip-0000000001size.hg + @@ -15,6 +16,7 @@ + 02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg + 02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg + 02-c12927fef661d2463043347101b90067c2961333-0000001280skip-0000000128size.hg + +02-c232505f58fdf70bcf5f6ab6a555f23ffc74f761-0000001523skip-0000000001size.hg + 02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg + 02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg + 02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001024skip-0000000227size.hg + @@ -24,6 +26,7 @@ + 02-dc714c3a5d080165292ba99b097567d0b95e5756-0000001408skip-0000000064size.hg + 02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg + 02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg + +02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001528skip-0000000008size.hg + 02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg + 02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg + 02-fb6c210a224903e81e5a8d2ee099cb0c9526ba8c-0000001512skip-0000000004size.hg + [1]