changegroup: add v4 changegroup for revlog v2 exchange
This change only adds the required infrastructure for the new changegroup format
and does not do any actual exchange. This will be done in the next patches.
Differential Revision: https://phab.mercurial-scm.org/D10026
--- a/hgext/remotefilelog/remotefilelog.py Fri Feb 19 11:07:10 2021 +0100
+++ b/hgext/remotefilelog/remotefilelog.py Thu Feb 18 17:36:52 2021 +0100
@@ -306,6 +306,7 @@
assumehaveparentrevisions=False,
deltaprevious=False,
deltamode=None,
+ sidedata_helpers=None,
):
# we don't use any of these parameters here
del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
@@ -333,6 +334,8 @@
baserevisionsize=None,
revision=revision,
delta=delta,
+ # Sidedata is not supported yet
+ sidedata=None,
)
def revdiff(self, node1, node2):
--- a/hgext/remotefilelog/shallowbundle.py Fri Feb 19 11:07:10 2021 +0100
+++ b/hgext/remotefilelog/shallowbundle.py Thu Feb 18 17:36:52 2021 +0100
@@ -67,7 +67,7 @@
shallowcg1packer, self, nodelist, rlog, lookup, units=units
)
- def generatefiles(self, changedfiles, *args):
+ def generatefiles(self, changedfiles, *args, **kwargs):
try:
linknodes, commonrevs, source = args
except ValueError:
@@ -92,7 +92,9 @@
[f for f in changedfiles if not repo.shallowmatch(f)]
)
- return super(shallowcg1packer, self).generatefiles(changedfiles, *args)
+ return super(shallowcg1packer, self).generatefiles(
+ changedfiles, *args, **kwargs
+ )
def shouldaddfilegroups(self, source):
repo = self._repo
@@ -176,9 +178,11 @@
repo.shallowmatch = original
-def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
+def addchangegroupfiles(
+ orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
+):
if not shallowutil.isenabled(repo):
- return orig(repo, source, revmap, trp, expectedfiles, *args)
+ return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
newfiles = 0
visited = set()
@@ -272,7 +276,7 @@
revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
- node, p1, p2, linknode, deltabase, delta, flags = revisiondata
+ node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata
if not available(f, node, f, deltabase):
continue
--- a/hgext/sqlitestore.py Fri Feb 19 11:07:10 2021 +0100
+++ b/hgext/sqlitestore.py Thu Feb 18 17:36:52 2021 +0100
@@ -681,7 +681,16 @@
):
empty = True
- for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
+ for (
+ node,
+ p1,
+ p2,
+ linknode,
+ deltabase,
+ delta,
+ wireflags,
+ sidedata,
+ ) in deltas:
storeflags = 0
if wireflags & repository.REVISION_FLAG_CENSORED:
--- a/mercurial/bundlerepo.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/bundlerepo.py Thu Feb 18 17:36:52 2021 +0100
@@ -61,7 +61,7 @@
self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression
for deltadata in cgunpacker.deltaiter():
- node, p1, p2, cs, deltabase, delta, flags = deltadata
+ node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
size = len(delta)
start = cgunpacker.tell() - size
--- a/mercurial/changegroup.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/changegroup.py Thu Feb 18 17:36:52 2021 +0100
@@ -32,6 +32,7 @@
)
from .interfaces import repository
+from .revlogutils import sidedata as sidedatamod
_CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
@@ -202,7 +203,9 @@
header = self.deltaheader.unpack(headerdata)
delta = readexactly(self._stream, l - self.deltaheadersize)
node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
- return (node, p1, p2, cs, deltabase, delta, flags)
+ # cg4 forward-compat
+ sidedata = {}
+ return (node, p1, p2, cs, deltabase, delta, flags, sidedata)
def getchunks(self):
"""returns all the chunks contains in the bundle
@@ -552,6 +555,29 @@
raise error.Abort(_(b"received dir revlog group is empty"))
+class cg4unpacker(cg3unpacker):
+ """Unpacker for cg4 streams.
+
+ cg4 streams add support for exchanging sidedata.
+ """
+
+ version = b'04'
+
+ def deltachunk(self, prevnode):
+ res = super(cg4unpacker, self).deltachunk(prevnode)
+ if not res:
+ return res
+
+ (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res
+
+ sidedata_raw = getchunk(self._stream)
+ sidedata = {}
+ if len(sidedata_raw) > 0:
+ sidedata = sidedatamod.deserialize_sidedata(sidedata_raw)
+
+ return node, p1, p2, cs, deltabase, delta, flags, sidedata
+
+
class headerlessfixup(object):
def __init__(self, fh, h):
self._h = h
@@ -861,6 +887,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
"""Given a source repo, construct a bundler.
@@ -893,6 +920,8 @@
nodes. We store this rather than the set of nodes that should be
ellipsis because for very large histories we expect this to be
significantly smaller.
+
+ remote_sidedata is the set of sidedata categories wanted by the remote.
"""
assert oldmatcher
assert matcher
@@ -988,7 +1017,7 @@
for tree, deltas in it:
if tree:
- assert self.version == b'03'
+ assert self.version in (b'03', b'04')
chunk = _fileheader(tree)
size += len(chunk)
yield chunk
@@ -1394,6 +1423,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.linknode
@@ -1424,6 +1454,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode
@@ -1453,6 +1484,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
@@ -1473,12 +1505,47 @@
)
+def _makecg4packer(
+ repo,
+ oldmatcher,
+ matcher,
+ bundlecaps,
+ ellipses=False,
+ shallow=False,
+ ellipsisroots=None,
+ fullnodes=None,
+ remote_sidedata=None,
+):
+ # Same header func as cg3. Sidedata is in a separate chunk from the delta to
+ # differenciate "raw delta" and sidedata.
+ builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
+ d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
+ )
+
+ return cgpacker(
+ repo,
+ oldmatcher,
+ matcher,
+ b'04',
+ builddeltaheader=builddeltaheader,
+ manifestsend=closechunk(),
+ bundlecaps=bundlecaps,
+ ellipses=ellipses,
+ shallow=shallow,
+ ellipsisroots=ellipsisroots,
+ fullnodes=fullnodes,
+ remote_sidedata=remote_sidedata,
+ )
+
+
_packermap = {
b'01': (_makecg1packer, cg1unpacker),
# cg2 adds support for exchanging generaldelta
b'02': (_makecg2packer, cg2unpacker),
# cg3 adds support for exchanging revlog flags and treemanifests
b'03': (_makecg3packer, cg3unpacker),
+ # ch4 adds support for exchanging sidedata
+ b'04': (_makecg4packer, cg4unpacker),
}
@@ -1498,11 +1565,9 @@
#
# (or even to push subset of history)
needv03 = True
- if b'exp-sidedata-flag' in repo.requirements:
- needv03 = True
- # don't attempt to use 01/02 until we do sidedata cleaning
- versions.discard(b'01')
- versions.discard(b'02')
+ has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements
+ if not has_revlogv2:
+ versions.discard(b'04')
if not needv03:
versions.discard(b'03')
return versions
@@ -1565,6 +1630,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
assert version in supportedoutgoingversions(repo)
@@ -1601,6 +1667,7 @@
shallow=shallow,
ellipsisroots=ellipsisroots,
fullnodes=fullnodes,
+ remote_sidedata=remote_sidedata,
)
@@ -1644,8 +1711,15 @@
fastpath=False,
bundlecaps=None,
matcher=None,
+ remote_sidedata=None,
):
- bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher)
+ bundler = getbundler(
+ version,
+ repo,
+ bundlecaps=bundlecaps,
+ matcher=matcher,
+ remote_sidedata=remote_sidedata,
+ )
repo = repo.unfiltered()
commonrevs = outgoing.common
--- a/mercurial/debugcommands.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/debugcommands.py Thu Feb 18 17:36:52 2021 +0100
@@ -346,7 +346,7 @@
def showchunks(named):
ui.write(b"\n%s%s\n" % (indent_string, named))
for deltadata in gen.deltaiter():
- node, p1, p2, cs, deltabase, delta, flags = deltadata
+ node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
ui.write(
b"%s%s %s %s %s %s %d\n"
% (
@@ -372,7 +372,7 @@
raise error.Abort(_(b'use debugbundle2 for this file'))
gen.changelogheader()
for deltadata in gen.deltaiter():
- node, p1, p2, cs, deltabase, delta, flags = deltadata
+ node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
ui.write(b"%s%s\n" % (indent_string, hex(node)))
--- a/mercurial/exchange.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/exchange.py Thu Feb 18 17:36:52 2021 +0100
@@ -2249,7 +2249,13 @@
def getbundlechunks(
- repo, source, heads=None, common=None, bundlecaps=None, **kwargs
+ repo,
+ source,
+ heads=None,
+ common=None,
+ bundlecaps=None,
+ remote_sidedata=None,
+ **kwargs
):
"""Return chunks constituting a bundle's raw data.
@@ -2279,7 +2285,12 @@
return (
info,
changegroup.makestream(
- repo, outgoing, b'01', source, bundlecaps=bundlecaps
+ repo,
+ outgoing,
+ b'01',
+ source,
+ bundlecaps=bundlecaps,
+ remote_sidedata=remote_sidedata,
),
)
@@ -2303,6 +2314,7 @@
source,
bundlecaps=bundlecaps,
b2caps=b2caps,
+ remote_sidedata=remote_sidedata,
**pycompat.strkwargs(kwargs)
)
@@ -2325,6 +2337,7 @@
b2caps=None,
heads=None,
common=None,
+ remote_sidedata=None,
**kwargs
):
"""add a changegroup part to the requested bundle"""
@@ -2355,7 +2368,13 @@
matcher = None
cgstream = changegroup.makestream(
- repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher
+ repo,
+ outgoing,
+ version,
+ source,
+ bundlecaps=bundlecaps,
+ matcher=matcher,
+ remote_sidedata=remote_sidedata,
)
part = bundler.newpart(b'changegroup', data=cgstream)
--- a/mercurial/exchangev2.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/exchangev2.py Thu Feb 18 17:36:52 2021 +0100
@@ -417,6 +417,8 @@
mdiff.trivialdiffheader(len(data)) + data,
# Flags not yet supported.
0,
+ # Sidedata not yet supported
+ {},
)
cl.addgroup(
@@ -496,6 +498,8 @@
delta,
# Flags not yet supported.
0,
+ # Sidedata not yet supported.
+ {},
)
progress.increment()
@@ -621,6 +625,8 @@
delta,
# Flags not yet supported.
0,
+ # Sidedata not yet supported.
+ {},
)
progress.increment()
@@ -719,6 +725,8 @@
delta,
# Flags not yet supported.
0,
+ # Sidedata not yet supported.
+ {},
)
progress.increment()
--- a/mercurial/localrepo.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/localrepo.py Thu Feb 18 17:36:52 2021 +0100
@@ -316,7 +316,13 @@
)
def getbundle(
- self, source, heads=None, common=None, bundlecaps=None, **kwargs
+ self,
+ source,
+ heads=None,
+ common=None,
+ bundlecaps=None,
+ remote_sidedata=None,
+ **kwargs
):
chunks = exchange.getbundlechunks(
self._repo,
@@ -324,6 +330,7 @@
heads=heads,
common=common,
bundlecaps=bundlecaps,
+ remote_sidedata=remote_sidedata,
**kwargs
)[1]
cb = util.chunkbuffer(chunks)
--- a/mercurial/revlog.py Fri Feb 19 11:07:10 2021 +0100
+++ b/mercurial/revlog.py Thu Feb 18 17:36:52 2021 +0100
@@ -2527,7 +2527,7 @@
deltacomputer = deltautil.deltacomputer(self)
# loop through our set of deltas
for data in deltas:
- node, p1, p2, linknode, deltabase, delta, flags = data
+ node, p1, p2, linknode, deltabase, delta, flags, sidedata = data
link = linkmapper(linknode)
flags = flags or REVIDX_DEFAULT_FLAGS
--- a/tests/test-check-interfaces.py Fri Feb 19 11:07:10 2021 +0100
+++ b/tests/test-check-interfaces.py Thu Feb 18 17:36:52 2021 +0100
@@ -276,6 +276,7 @@
flags=b'',
baserevisionsize=None,
revision=b'',
+ sidedata=b'',
delta=None,
)
checkzobject(rd)
--- a/tests/test-revlog-raw.py Fri Feb 19 11:07:10 2021 +0100
+++ b/tests/test-revlog-raw.py Thu Feb 18 17:36:52 2021 +0100
@@ -147,6 +147,7 @@
b'flags': rlog.flags(r),
b'deltabase': rlog.node(deltaparent),
b'delta': rlog.revdiff(deltaparent, r),
+ b'sidedata': rlog.sidedata(r),
}
def deltaiter(self):
@@ -159,10 +160,11 @@
deltabase = chunkdata[b'deltabase']
delta = chunkdata[b'delta']
flags = chunkdata[b'flags']
+ sidedata = chunkdata[b'sidedata']
chain = node
- yield (node, p1, p2, cs, deltabase, delta, flags)
+ yield (node, p1, p2, cs, deltabase, delta, flags, sidedata)
def linkmap(lnode):
return rlog.rev(lnode)