changeset 46711:a41565bef69f

changegroup: add v4 changegroup for revlog v2 exchange This change only adds the required infrastructure for the new changegroup format and does not do any actual exchange. This will be done in the next patches. Differential Revision: https://phab.mercurial-scm.org/D10026
author Raphaël Gomès <rgomes@octobus.net>
date Thu, 18 Feb 2021 17:36:52 +0100
parents 4cd214c9948d
children e8c11a2c96c0
files hgext/remotefilelog/remotefilelog.py hgext/remotefilelog/shallowbundle.py hgext/sqlitestore.py mercurial/bundlerepo.py mercurial/changegroup.py mercurial/debugcommands.py mercurial/exchange.py mercurial/exchangev2.py mercurial/localrepo.py mercurial/revlog.py tests/test-check-interfaces.py tests/test-revlog-raw.py
diffstat 12 files changed, 150 insertions(+), 23 deletions(-) [+]
line wrap: on
line diff
--- a/hgext/remotefilelog/remotefilelog.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/hgext/remotefilelog/remotefilelog.py	Thu Feb 18 17:36:52 2021 +0100
@@ -306,6 +306,7 @@
         assumehaveparentrevisions=False,
         deltaprevious=False,
         deltamode=None,
+        sidedata_helpers=None,
     ):
         # we don't use any of these parameters here
         del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
@@ -333,6 +334,8 @@
                 baserevisionsize=None,
                 revision=revision,
                 delta=delta,
+                # Sidedata is not supported yet
+                sidedata=None,
             )
 
     def revdiff(self, node1, node2):
--- a/hgext/remotefilelog/shallowbundle.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/hgext/remotefilelog/shallowbundle.py	Thu Feb 18 17:36:52 2021 +0100
@@ -67,7 +67,7 @@
             shallowcg1packer, self, nodelist, rlog, lookup, units=units
         )
 
-    def generatefiles(self, changedfiles, *args):
+    def generatefiles(self, changedfiles, *args, **kwargs):
         try:
             linknodes, commonrevs, source = args
         except ValueError:
@@ -92,7 +92,9 @@
                     [f for f in changedfiles if not repo.shallowmatch(f)]
                 )
 
-        return super(shallowcg1packer, self).generatefiles(changedfiles, *args)
+        return super(shallowcg1packer, self).generatefiles(
+            changedfiles, *args, **kwargs
+        )
 
     def shouldaddfilegroups(self, source):
         repo = self._repo
@@ -176,9 +178,11 @@
         repo.shallowmatch = original
 
 
-def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
+def addchangegroupfiles(
+    orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
+):
     if not shallowutil.isenabled(repo):
-        return orig(repo, source, revmap, trp, expectedfiles, *args)
+        return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
 
     newfiles = 0
     visited = set()
@@ -272,7 +276,7 @@
 
         revisiondata = revisiondatas[(f, node)]
         # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
-        node, p1, p2, linknode, deltabase, delta, flags = revisiondata
+        node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata
 
         if not available(f, node, f, deltabase):
             continue
--- a/hgext/sqlitestore.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/hgext/sqlitestore.py	Thu Feb 18 17:36:52 2021 +0100
@@ -681,7 +681,16 @@
     ):
         empty = True
 
-        for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
+        for (
+            node,
+            p1,
+            p2,
+            linknode,
+            deltabase,
+            delta,
+            wireflags,
+            sidedata,
+        ) in deltas:
             storeflags = 0
 
             if wireflags & repository.REVISION_FLAG_CENSORED:
--- a/mercurial/bundlerepo.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/bundlerepo.py	Thu Feb 18 17:36:52 2021 +0100
@@ -61,7 +61,7 @@
         self.repotiprev = n - 1
         self.bundlerevs = set()  # used by 'bundle()' revset expression
         for deltadata in cgunpacker.deltaiter():
-            node, p1, p2, cs, deltabase, delta, flags = deltadata
+            node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
 
             size = len(delta)
             start = cgunpacker.tell() - size
--- a/mercurial/changegroup.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/changegroup.py	Thu Feb 18 17:36:52 2021 +0100
@@ -32,6 +32,7 @@
 )
 
 from .interfaces import repository
+from .revlogutils import sidedata as sidedatamod
 
 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
@@ -202,7 +203,9 @@
         header = self.deltaheader.unpack(headerdata)
         delta = readexactly(self._stream, l - self.deltaheadersize)
         node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
-        return (node, p1, p2, cs, deltabase, delta, flags)
+        # cg4 forward-compat
+        sidedata = {}
+        return (node, p1, p2, cs, deltabase, delta, flags, sidedata)
 
     def getchunks(self):
         """returns all the chunks contains in the bundle
@@ -552,6 +555,29 @@
                 raise error.Abort(_(b"received dir revlog group is empty"))
 
 
+class cg4unpacker(cg3unpacker):
+    """Unpacker for cg4 streams.
+
+    cg4 streams add support for exchanging sidedata.
+    """
+
+    version = b'04'
+
+    def deltachunk(self, prevnode):
+        res = super(cg4unpacker, self).deltachunk(prevnode)
+        if not res:
+            return res
+
+        (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res
+
+        sidedata_raw = getchunk(self._stream)
+        sidedata = {}
+        if len(sidedata_raw) > 0:
+            sidedata = sidedatamod.deserialize_sidedata(sidedata_raw)
+
+        return node, p1, p2, cs, deltabase, delta, flags, sidedata
+
+
 class headerlessfixup(object):
     def __init__(self, fh, h):
         self._h = h
@@ -861,6 +887,7 @@
         shallow=False,
         ellipsisroots=None,
         fullnodes=None,
+        remote_sidedata=None,
     ):
         """Given a source repo, construct a bundler.
 
@@ -893,6 +920,8 @@
         nodes. We store this rather than the set of nodes that should be
         ellipsis because for very large histories we expect this to be
         significantly smaller.
+
+        remote_sidedata is the set of sidedata categories wanted by the remote.
         """
         assert oldmatcher
         assert matcher
@@ -988,7 +1017,7 @@
 
         for tree, deltas in it:
             if tree:
-                assert self.version == b'03'
+                assert self.version in (b'03', b'04')
                 chunk = _fileheader(tree)
                 size += len(chunk)
                 yield chunk
@@ -1394,6 +1423,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.linknode
@@ -1424,6 +1454,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode
@@ -1453,6 +1484,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
@@ -1473,12 +1505,47 @@
     )
 
 
+def _makecg4packer(
+    repo,
+    oldmatcher,
+    matcher,
+    bundlecaps,
+    ellipses=False,
+    shallow=False,
+    ellipsisroots=None,
+    fullnodes=None,
+    remote_sidedata=None,
+):
+    # Same header func as cg3. Sidedata is in a separate chunk from the delta to
+    # differenciate "raw delta" and sidedata.
+    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
+        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
+    )
+
+    return cgpacker(
+        repo,
+        oldmatcher,
+        matcher,
+        b'04',
+        builddeltaheader=builddeltaheader,
+        manifestsend=closechunk(),
+        bundlecaps=bundlecaps,
+        ellipses=ellipses,
+        shallow=shallow,
+        ellipsisroots=ellipsisroots,
+        fullnodes=fullnodes,
+        remote_sidedata=remote_sidedata,
+    )
+
+
 _packermap = {
     b'01': (_makecg1packer, cg1unpacker),
     # cg2 adds support for exchanging generaldelta
     b'02': (_makecg2packer, cg2unpacker),
     # cg3 adds support for exchanging revlog flags and treemanifests
     b'03': (_makecg3packer, cg3unpacker),
+    # ch4 adds support for exchanging sidedata
+    b'04': (_makecg4packer, cg4unpacker),
 }
 
 
@@ -1498,11 +1565,9 @@
         #
         # (or even to push subset of history)
         needv03 = True
-    if b'exp-sidedata-flag' in repo.requirements:
-        needv03 = True
-        # don't attempt to use 01/02 until we do sidedata cleaning
-        versions.discard(b'01')
-        versions.discard(b'02')
+    has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements
+    if not has_revlogv2:
+        versions.discard(b'04')
     if not needv03:
         versions.discard(b'03')
     return versions
@@ -1565,6 +1630,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     assert version in supportedoutgoingversions(repo)
 
@@ -1601,6 +1667,7 @@
         shallow=shallow,
         ellipsisroots=ellipsisroots,
         fullnodes=fullnodes,
+        remote_sidedata=remote_sidedata,
     )
 
 
@@ -1644,8 +1711,15 @@
     fastpath=False,
     bundlecaps=None,
     matcher=None,
+    remote_sidedata=None,
 ):
-    bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher)
+    bundler = getbundler(
+        version,
+        repo,
+        bundlecaps=bundlecaps,
+        matcher=matcher,
+        remote_sidedata=remote_sidedata,
+    )
 
     repo = repo.unfiltered()
     commonrevs = outgoing.common
--- a/mercurial/debugcommands.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/debugcommands.py	Thu Feb 18 17:36:52 2021 +0100
@@ -346,7 +346,7 @@
         def showchunks(named):
             ui.write(b"\n%s%s\n" % (indent_string, named))
             for deltadata in gen.deltaiter():
-                node, p1, p2, cs, deltabase, delta, flags = deltadata
+                node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
                 ui.write(
                     b"%s%s %s %s %s %s %d\n"
                     % (
@@ -372,7 +372,7 @@
             raise error.Abort(_(b'use debugbundle2 for this file'))
         gen.changelogheader()
         for deltadata in gen.deltaiter():
-            node, p1, p2, cs, deltabase, delta, flags = deltadata
+            node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
             ui.write(b"%s%s\n" % (indent_string, hex(node)))
 
 
--- a/mercurial/exchange.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/exchange.py	Thu Feb 18 17:36:52 2021 +0100
@@ -2249,7 +2249,13 @@
 
 
 def getbundlechunks(
-    repo, source, heads=None, common=None, bundlecaps=None, **kwargs
+    repo,
+    source,
+    heads=None,
+    common=None,
+    bundlecaps=None,
+    remote_sidedata=None,
+    **kwargs
 ):
     """Return chunks constituting a bundle's raw data.
 
@@ -2279,7 +2285,12 @@
         return (
             info,
             changegroup.makestream(
-                repo, outgoing, b'01', source, bundlecaps=bundlecaps
+                repo,
+                outgoing,
+                b'01',
+                source,
+                bundlecaps=bundlecaps,
+                remote_sidedata=remote_sidedata,
             ),
         )
 
@@ -2303,6 +2314,7 @@
             source,
             bundlecaps=bundlecaps,
             b2caps=b2caps,
+            remote_sidedata=remote_sidedata,
             **pycompat.strkwargs(kwargs)
         )
 
@@ -2325,6 +2337,7 @@
     b2caps=None,
     heads=None,
     common=None,
+    remote_sidedata=None,
     **kwargs
 ):
     """add a changegroup part to the requested bundle"""
@@ -2355,7 +2368,13 @@
         matcher = None
 
     cgstream = changegroup.makestream(
-        repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher
+        repo,
+        outgoing,
+        version,
+        source,
+        bundlecaps=bundlecaps,
+        matcher=matcher,
+        remote_sidedata=remote_sidedata,
     )
 
     part = bundler.newpart(b'changegroup', data=cgstream)
--- a/mercurial/exchangev2.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/exchangev2.py	Thu Feb 18 17:36:52 2021 +0100
@@ -417,6 +417,8 @@
                 mdiff.trivialdiffheader(len(data)) + data,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported
+                {},
             )
 
     cl.addgroup(
@@ -496,6 +498,8 @@
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
@@ -621,6 +625,8 @@
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
@@ -719,6 +725,8 @@
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
--- a/mercurial/localrepo.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/localrepo.py	Thu Feb 18 17:36:52 2021 +0100
@@ -316,7 +316,13 @@
         )
 
     def getbundle(
-        self, source, heads=None, common=None, bundlecaps=None, **kwargs
+        self,
+        source,
+        heads=None,
+        common=None,
+        bundlecaps=None,
+        remote_sidedata=None,
+        **kwargs
     ):
         chunks = exchange.getbundlechunks(
             self._repo,
@@ -324,6 +330,7 @@
             heads=heads,
             common=common,
             bundlecaps=bundlecaps,
+            remote_sidedata=remote_sidedata,
             **kwargs
         )[1]
         cb = util.chunkbuffer(chunks)
--- a/mercurial/revlog.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/revlog.py	Thu Feb 18 17:36:52 2021 +0100
@@ -2527,7 +2527,7 @@
             deltacomputer = deltautil.deltacomputer(self)
             # loop through our set of deltas
             for data in deltas:
-                node, p1, p2, linknode, deltabase, delta, flags = data
+                node, p1, p2, linknode, deltabase, delta, flags, sidedata = data
                 link = linkmapper(linknode)
                 flags = flags or REVIDX_DEFAULT_FLAGS
 
--- a/tests/test-check-interfaces.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/tests/test-check-interfaces.py	Thu Feb 18 17:36:52 2021 +0100
@@ -276,6 +276,7 @@
         flags=b'',
         baserevisionsize=None,
         revision=b'',
+        sidedata=b'',
         delta=None,
     )
     checkzobject(rd)
--- a/tests/test-revlog-raw.py	Fri Feb 19 11:07:10 2021 +0100
+++ b/tests/test-revlog-raw.py	Thu Feb 18 17:36:52 2021 +0100
@@ -147,6 +147,7 @@
                 b'flags': rlog.flags(r),
                 b'deltabase': rlog.node(deltaparent),
                 b'delta': rlog.revdiff(deltaparent, r),
+                b'sidedata': rlog.sidedata(r),
             }
 
         def deltaiter(self):
@@ -159,10 +160,11 @@
                 deltabase = chunkdata[b'deltabase']
                 delta = chunkdata[b'delta']
                 flags = chunkdata[b'flags']
+                sidedata = chunkdata[b'sidedata']
 
                 chain = node
 
-                yield (node, p1, p2, cs, deltabase, delta, flags)
+                yield (node, p1, p2, cs, deltabase, delta, flags, sidedata)
 
     def linkmap(lnode):
         return rlog.rev(lnode)