Mercurial > hg
changeset 46711:a41565bef69f
changegroup: add v4 changegroup for revlog v2 exchange
This change only adds the required infrastructure for the new changegroup format
and does not do any actual exchange. This will be done in the next patches.
Differential Revision: https://phab.mercurial-scm.org/D10026
| author | Raphaël Gomès <rgomes@octobus.net> |
|---|---|
| date | Thu, 18 Feb 2021 17:36:52 +0100 |
| parents | 4cd214c9948d |
| children | e8c11a2c96c0 |
| files | hgext/remotefilelog/remotefilelog.py hgext/remotefilelog/shallowbundle.py hgext/sqlitestore.py mercurial/bundlerepo.py mercurial/changegroup.py mercurial/debugcommands.py mercurial/exchange.py mercurial/exchangev2.py mercurial/localrepo.py mercurial/revlog.py tests/test-check-interfaces.py tests/test-revlog-raw.py |
| diffstat | 12 files changed, 150 insertions(+), 23 deletions(-) [+] |
line wrap: on
line diff
--- a/hgext/remotefilelog/remotefilelog.py Fri Feb 19 11:07:10 2021 +0100 +++ b/hgext/remotefilelog/remotefilelog.py Thu Feb 18 17:36:52 2021 +0100 @@ -306,6 +306,7 @@ assumehaveparentrevisions=False, deltaprevious=False, deltamode=None, + sidedata_helpers=None, ): # we don't use any of these parameters here del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious @@ -333,6 +334,8 @@ baserevisionsize=None, revision=revision, delta=delta, + # Sidedata is not supported yet + sidedata=None, ) def revdiff(self, node1, node2):
--- a/hgext/remotefilelog/shallowbundle.py Fri Feb 19 11:07:10 2021 +0100 +++ b/hgext/remotefilelog/shallowbundle.py Thu Feb 18 17:36:52 2021 +0100 @@ -67,7 +67,7 @@ shallowcg1packer, self, nodelist, rlog, lookup, units=units ) - def generatefiles(self, changedfiles, *args): + def generatefiles(self, changedfiles, *args, **kwargs): try: linknodes, commonrevs, source = args except ValueError: @@ -92,7 +92,9 @@ [f for f in changedfiles if not repo.shallowmatch(f)] ) - return super(shallowcg1packer, self).generatefiles(changedfiles, *args) + return super(shallowcg1packer, self).generatefiles( + changedfiles, *args, **kwargs + ) def shouldaddfilegroups(self, source): repo = self._repo @@ -176,9 +178,11 @@ repo.shallowmatch = original -def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args): +def addchangegroupfiles( + orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs +): if not shallowutil.isenabled(repo): - return orig(repo, source, revmap, trp, expectedfiles, *args) + return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs) newfiles = 0 visited = set() @@ -272,7 +276,7 @@ revisiondata = revisiondatas[(f, node)] # revisiondata: (node, p1, p2, cs, deltabase, delta, flags) - node, p1, p2, linknode, deltabase, delta, flags = revisiondata + node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata if not available(f, node, f, deltabase): continue
--- a/hgext/sqlitestore.py Fri Feb 19 11:07:10 2021 +0100 +++ b/hgext/sqlitestore.py Thu Feb 18 17:36:52 2021 +0100 @@ -681,7 +681,16 @@ ): empty = True - for node, p1, p2, linknode, deltabase, delta, wireflags in deltas: + for ( + node, + p1, + p2, + linknode, + deltabase, + delta, + wireflags, + sidedata, + ) in deltas: storeflags = 0 if wireflags & repository.REVISION_FLAG_CENSORED:
--- a/mercurial/bundlerepo.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/bundlerepo.py Thu Feb 18 17:36:52 2021 +0100 @@ -61,7 +61,7 @@ self.repotiprev = n - 1 self.bundlerevs = set() # used by 'bundle()' revset expression for deltadata in cgunpacker.deltaiter(): - node, p1, p2, cs, deltabase, delta, flags = deltadata + node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata size = len(delta) start = cgunpacker.tell() - size
--- a/mercurial/changegroup.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/changegroup.py Thu Feb 18 17:36:52 2021 +0100 @@ -32,6 +32,7 @@ ) from .interfaces import repository +from .revlogutils import sidedata as sidedatamod _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s") _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s") @@ -202,7 +203,9 @@ header = self.deltaheader.unpack(headerdata) delta = readexactly(self._stream, l - self.deltaheadersize) node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) - return (node, p1, p2, cs, deltabase, delta, flags) + # cg4 forward-compat + sidedata = {} + return (node, p1, p2, cs, deltabase, delta, flags, sidedata) def getchunks(self): """returns all the chunks contains in the bundle @@ -552,6 +555,29 @@ raise error.Abort(_(b"received dir revlog group is empty")) +class cg4unpacker(cg3unpacker): + """Unpacker for cg4 streams. + + cg4 streams add support for exchanging sidedata. + """ + + version = b'04' + + def deltachunk(self, prevnode): + res = super(cg4unpacker, self).deltachunk(prevnode) + if not res: + return res + + (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res + + sidedata_raw = getchunk(self._stream) + sidedata = {} + if len(sidedata_raw) > 0: + sidedata = sidedatamod.deserialize_sidedata(sidedata_raw) + + return node, p1, p2, cs, deltabase, delta, flags, sidedata + + class headerlessfixup(object): def __init__(self, fh, h): self._h = h @@ -861,6 +887,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): """Given a source repo, construct a bundler. @@ -893,6 +920,8 @@ nodes. We store this rather than the set of nodes that should be ellipsis because for very large histories we expect this to be significantly smaller. + + remote_sidedata is the set of sidedata categories wanted by the remote. """ assert oldmatcher assert matcher @@ -988,7 +1017,7 @@ for tree, deltas in it: if tree: - assert self.version == b'03' + assert self.version in (b'03', b'04') chunk = _fileheader(tree) size += len(chunk) yield chunk @@ -1394,6 +1423,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.linknode @@ -1424,6 +1454,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.basenode, d.linknode @@ -1453,6 +1484,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags @@ -1473,12 +1505,47 @@ ) +def _makecg4packer( + repo, + oldmatcher, + matcher, + bundlecaps, + ellipses=False, + shallow=False, + ellipsisroots=None, + fullnodes=None, + remote_sidedata=None, +): + # Same header func as cg3. Sidedata is in a separate chunk from the delta to + # differenciate "raw delta" and sidedata. + builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( + d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags + ) + + return cgpacker( + repo, + oldmatcher, + matcher, + b'04', + builddeltaheader=builddeltaheader, + manifestsend=closechunk(), + bundlecaps=bundlecaps, + ellipses=ellipses, + shallow=shallow, + ellipsisroots=ellipsisroots, + fullnodes=fullnodes, + remote_sidedata=remote_sidedata, + ) + + _packermap = { b'01': (_makecg1packer, cg1unpacker), # cg2 adds support for exchanging generaldelta b'02': (_makecg2packer, cg2unpacker), # cg3 adds support for exchanging revlog flags and treemanifests b'03': (_makecg3packer, cg3unpacker), + # ch4 adds support for exchanging sidedata + b'04': (_makecg4packer, cg4unpacker), } @@ -1498,11 +1565,9 @@ # # (or even to push subset of history) needv03 = True - if b'exp-sidedata-flag' in repo.requirements: - needv03 = True - # don't attempt to use 01/02 until we do sidedata cleaning - versions.discard(b'01') - versions.discard(b'02') + has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements + if not has_revlogv2: + versions.discard(b'04') if not needv03: versions.discard(b'03') return versions @@ -1565,6 +1630,7 @@ shallow=False, ellipsisroots=None, fullnodes=None, + remote_sidedata=None, ): assert version in supportedoutgoingversions(repo) @@ -1601,6 +1667,7 @@ shallow=shallow, ellipsisroots=ellipsisroots, fullnodes=fullnodes, + remote_sidedata=remote_sidedata, ) @@ -1644,8 +1711,15 @@ fastpath=False, bundlecaps=None, matcher=None, + remote_sidedata=None, ): - bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher) + bundler = getbundler( + version, + repo, + bundlecaps=bundlecaps, + matcher=matcher, + remote_sidedata=remote_sidedata, + ) repo = repo.unfiltered() commonrevs = outgoing.common
--- a/mercurial/debugcommands.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/debugcommands.py Thu Feb 18 17:36:52 2021 +0100 @@ -346,7 +346,7 @@ def showchunks(named): ui.write(b"\n%s%s\n" % (indent_string, named)) for deltadata in gen.deltaiter(): - node, p1, p2, cs, deltabase, delta, flags = deltadata + node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata ui.write( b"%s%s %s %s %s %s %d\n" % ( @@ -372,7 +372,7 @@ raise error.Abort(_(b'use debugbundle2 for this file')) gen.changelogheader() for deltadata in gen.deltaiter(): - node, p1, p2, cs, deltabase, delta, flags = deltadata + node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata ui.write(b"%s%s\n" % (indent_string, hex(node)))
--- a/mercurial/exchange.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/exchange.py Thu Feb 18 17:36:52 2021 +0100 @@ -2249,7 +2249,13 @@ def getbundlechunks( - repo, source, heads=None, common=None, bundlecaps=None, **kwargs + repo, + source, + heads=None, + common=None, + bundlecaps=None, + remote_sidedata=None, + **kwargs ): """Return chunks constituting a bundle's raw data. @@ -2279,7 +2285,12 @@ return ( info, changegroup.makestream( - repo, outgoing, b'01', source, bundlecaps=bundlecaps + repo, + outgoing, + b'01', + source, + bundlecaps=bundlecaps, + remote_sidedata=remote_sidedata, ), ) @@ -2303,6 +2314,7 @@ source, bundlecaps=bundlecaps, b2caps=b2caps, + remote_sidedata=remote_sidedata, **pycompat.strkwargs(kwargs) ) @@ -2325,6 +2337,7 @@ b2caps=None, heads=None, common=None, + remote_sidedata=None, **kwargs ): """add a changegroup part to the requested bundle""" @@ -2355,7 +2368,13 @@ matcher = None cgstream = changegroup.makestream( - repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher + repo, + outgoing, + version, + source, + bundlecaps=bundlecaps, + matcher=matcher, + remote_sidedata=remote_sidedata, ) part = bundler.newpart(b'changegroup', data=cgstream)
--- a/mercurial/exchangev2.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/exchangev2.py Thu Feb 18 17:36:52 2021 +0100 @@ -417,6 +417,8 @@ mdiff.trivialdiffheader(len(data)) + data, # Flags not yet supported. 0, + # Sidedata not yet supported + {}, ) cl.addgroup( @@ -496,6 +498,8 @@ delta, # Flags not yet supported. 0, + # Sidedata not yet supported. + {}, ) progress.increment() @@ -621,6 +625,8 @@ delta, # Flags not yet supported. 0, + # Sidedata not yet supported. + {}, ) progress.increment() @@ -719,6 +725,8 @@ delta, # Flags not yet supported. 0, + # Sidedata not yet supported. + {}, ) progress.increment()
--- a/mercurial/localrepo.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/localrepo.py Thu Feb 18 17:36:52 2021 +0100 @@ -316,7 +316,13 @@ ) def getbundle( - self, source, heads=None, common=None, bundlecaps=None, **kwargs + self, + source, + heads=None, + common=None, + bundlecaps=None, + remote_sidedata=None, + **kwargs ): chunks = exchange.getbundlechunks( self._repo, @@ -324,6 +330,7 @@ heads=heads, common=common, bundlecaps=bundlecaps, + remote_sidedata=remote_sidedata, **kwargs )[1] cb = util.chunkbuffer(chunks)
--- a/mercurial/revlog.py Fri Feb 19 11:07:10 2021 +0100 +++ b/mercurial/revlog.py Thu Feb 18 17:36:52 2021 +0100 @@ -2527,7 +2527,7 @@ deltacomputer = deltautil.deltacomputer(self) # loop through our set of deltas for data in deltas: - node, p1, p2, linknode, deltabase, delta, flags = data + node, p1, p2, linknode, deltabase, delta, flags, sidedata = data link = linkmapper(linknode) flags = flags or REVIDX_DEFAULT_FLAGS
--- a/tests/test-check-interfaces.py Fri Feb 19 11:07:10 2021 +0100 +++ b/tests/test-check-interfaces.py Thu Feb 18 17:36:52 2021 +0100 @@ -276,6 +276,7 @@ flags=b'', baserevisionsize=None, revision=b'', + sidedata=b'', delta=None, ) checkzobject(rd)
--- a/tests/test-revlog-raw.py Fri Feb 19 11:07:10 2021 +0100 +++ b/tests/test-revlog-raw.py Thu Feb 18 17:36:52 2021 +0100 @@ -147,6 +147,7 @@ b'flags': rlog.flags(r), b'deltabase': rlog.node(deltaparent), b'delta': rlog.revdiff(deltaparent, r), + b'sidedata': rlog.sidedata(r), } def deltaiter(self): @@ -159,10 +160,11 @@ deltabase = chunkdata[b'deltabase'] delta = chunkdata[b'delta'] flags = chunkdata[b'flags'] + sidedata = chunkdata[b'sidedata'] chain = node - yield (node, p1, p2, cs, deltabase, delta, flags) + yield (node, p1, p2, cs, deltabase, delta, flags, sidedata) def linkmap(lnode): return rlog.rev(lnode)