changeset 2243:d83851f2d375

obshashrange: have an half descent wireprotocol command

The previous implementation was extremely hacky. The new version is based on the
other discovery function and work!

--- a/README	Fri Mar 24 18:37:03 2017 +0100
+++ b/README	Fri Mar 24 18:21:48 2017 +0100
@@ -125,6 +125,21 @@
   Using the extension will enable evolution, use 'experimental.evolution=!'
   to disable obsmarkers echange.  The old '__temporary__.advertiseobsolete'
   option is no longer supported.
+
+- a new prototype of obsmarker discovery is available. The prototype is still
+  at early stage and not recommended for production.
+  Examples of current limitations:
+
+  - write access to the repo is highly recommanded for all operation,
+  - large memory footprint,
+  - initial caching is slow,
+  - unusable on large repo (because of various issue pointed earlier),
+  - likely to constains various bugs.
+
+  It can be tested by setting `experimental.obshashrange=1` on both client and
+  server. It is recommanded to get in touch with the evolve maintainer if you
+  decide to test it.
+
 - the 'debugrecordpruneparents' have been moved into the 'evolve.legacy'
   separate extension. enable that extentions if you need to convert/update
   markers in an old repository.

--- a/hgext3rd/evolve/obsdiscovery.py	Fri Mar 24 18:37:03 2017 +0100
+++ b/hgext3rd/evolve/obsdiscovery.py	Fri Mar 24 18:21:48 2017 +0100
@@ -29,7 +29,6 @@
 import weakref
 
 from mercurial import (
-    bundle2,
     dagutil,
     error,
     exchange,
@@ -53,6 +52,7 @@
 
 _pack = struct.pack
 _unpack = struct.unpack
+_calcsize = struct.calcsize
 
 eh = exthelper.exthelper()
 eh.merge(stablerange.eh)
@@ -153,32 +153,6 @@
 ###  Code performing discovery ###
 ##################################
 
-def _canobshashrange(local, remote):
-    return (local.ui.configbool('experimental', 'obshashrange', False)
-            and remote.capable('_donotusemeever_evoext_obshashrange_1'))
-
-
-def _obshashrange_capabilities(orig, repo, proto):
-    """wrapper to advertise new capability"""
-    caps = orig(repo, proto)
-    enabled = repo.ui.configbool('experimental', 'obshashrange', False)
-    if obsolete.isenabled(repo, obsolete.exchangeopt) and enabled:
-        caps = caps.split()
-        caps.append('_donotusemeever_evoext_obshashrange_1')
-        caps.sort()
-        caps = ' '.join(caps)
-    return caps
-
[e-mail]
-def obshashrange_extsetup(ui):
-    extensions.wrapfunction(wireproto, 'capabilities', _obshashrange_capabilities)
-    # wrap command content
-    oldcap, args = wireproto.commands['capabilities']
-
-    def newcap(repo, proto):
-        return _obshashrange_capabilities(oldcap, repo, proto)
-    wireproto.commands['capabilities'] = (newcap, args)
-
 def findcommonobsmarkers(ui, local, remote, probeset,
                          initialsamplesize=100,
                          fullsamplesize=200):
@@ -329,65 +303,14 @@
     return sorted(missing)
 
 def _queryrange(ui, repo, remote, allentries):
-    mapping = {}
+    #  question are asked with node
     n = repo.changelog.node
-    nodeentries = [(n(entry[0]), entry[1], entry) for entry in allentries]
-
-    def gen():
-        for enode, eindex, entry in nodeentries:
-            key = enode + _pack('>I', eindex)
-            mapping[key] = entry
-            yield key
-
-    bundler = bundle2.bundle20(ui, bundle2.bundle2caps(remote))
-    capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo))
-    bundler.newpart('replycaps', data=capsblob)
-    bundler.newpart('_donotusemeever_evoext_obshashrange_1', data=gen())
-
-    stream = util.chunkbuffer(bundler.getchunks())
-    try:
-        reply = remote.unbundle(
-            stream, ['force'], remote.url())
-    except error.BundleValueError as exc:
-        raise error.Abort(_('missing support for %s') % exc)
-    try:
-        op = bundle2.processbundle(repo, reply)
-    except error.BundleValueError as exc:
-        raise error.Abort(_('missing support for %s') % exc)
-    except bundle2.AbortFromPart as exc:
-        ui.status(_('remote: %s\n') % exc)
-        if exc.hint is not None:
-            ui.status(_('remote: %s\n') % ('(%s)' % exc.hint))
-        raise error.Abort(_('push failed on remote'))
-    for rep in op.records['_donotusemeever_evoext_obshashrange_1']:
-        yield mapping[rep['key']], rep['value']
-
-
[e-mail]('_donotusemeever_evoext_obshashrange_1', ())
-def _processqueryrange(op, inpart):
-    assert op.reply is not None
-    op.repo.stablerange.warmup(op.repo)
-    replies = []
-    data = inpart.read(24)
-    while data:
-        n = data[:20]
-        index = _unpack('>I', data[20:])[0]
-        r = op.repo.changelog.rev(n)
-        rhash = _obshashrange(op.repo, (r, index))
-        replies.append(data + rhash)
-        data = inpart.read(24)
-    op.repo.obsstore.rangeobshashcache.save(op.repo)
-    op.reply.newpart('reply:_donotusemeever_evoext_obshashrange_1', data=iter(replies))
-
-
[e-mail]('reply:_donotusemeever_evoext_obshashrange_1', ())
-def _processqueryrangereply(op, inpart):
-    data = inpart.read(44)
-    while data:
-        key = data[:24]
-        rhash = data[24:]
-        op.records.add('_donotusemeever_evoext_obshashrange_1', {'key': key, 'value': rhash})
-        data = inpart.read(44)
+    noderanges = [(n(entry[0]), entry[1]) for entry in allentries]
+    replies = remote.evoext_obshashrange_v0(noderanges)
+    result = []
+    for idx, entry in enumerate(allentries):
+        result.append((entry, replies[idx]))
+    return result
 
 ##############################
 ### Range Hash computation ###
@@ -620,6 +543,94 @@
 
     repo.__class__ = obshashrepo
 
+### wire protocol commands
+
+def _obshashrange_v0(repo, ranges):
+    """return a list of hash from a list of range
+
+    The range have the id encoded as a node
+
+    return 'wdirid' for unknown range"""
+    nm = repo.changelog.nodemap
+    ranges = [(nm.get(n), idx) for n, idx in ranges]
+    if ranges:
+        maxrev = max(r for r, i in ranges)
+        if maxrev is not None:
+            repo.stablerange.warmup(repo, upto=maxrev)
+    result = []
+    for r in ranges:
+        if r[0] is None:
+            result.append(node.wdirid)
+        else:
+            result.append(_obshashrange(repo, r))
+    repo.obsstore.rangeobshashcache.save(repo)
+    return result
+
[e-mail](localrepo.localpeer, 'evoext_obshashrange_v0')
+def local_obshashrange_v0(peer, ranges):
+    return _obshashrange_v0(peer._repo, ranges)
+
+
+_indexformat = '>I'
+_indexsize = _calcsize(_indexformat)
+def _encrange(node_rangeid):
+    """encode a (node) range"""
+    headnode, index = node_rangeid
+    return headnode + _pack(_indexformat, index)
+
+def _decrange(data):
+    """encode a (node) range"""
+    assert _indexsize < len(data), len(data)
+    headnode = data[:-_indexsize]
+    index = _unpack(_indexformat, data[-_indexsize:])[0]
+    return (headnode, index)
+
[e-mail](wireproto.wirepeer, 'evoext_obshashrange_v0')
+def peer_obshashrange_v0(self, ranges):
+    binranges = [_encrange(r) for r in ranges]
+    encranges = wireproto.encodelist(binranges)
+    d = self._call("evoext_obshashrange_v0", ranges=encranges)
+    try:
+        return wireproto.decodelist(d)
+    except ValueError:
+        self._abort(error.ResponseError(_("unexpected response:"), d))
+
+def srv_obshashrange_v0(repo, proto, ranges):
+    ranges = wireproto.decodelist(ranges)
+    ranges = [_decrange(r) for r in ranges]
+    hashes = _obshashrange_v0(repo, ranges)
+    return wireproto.encodelist(hashes)
+
+
+def _canobshashrange(local, remote):
+    return (local.ui.configbool('experimental', 'obshashrange', False)
+            and remote.capable('_evoext_obshashrange_v0'))
+
+def _obshashrange_capabilities(orig, repo, proto):
+    """wrapper to advertise new capability"""
+    caps = orig(repo, proto)
+    enabled = repo.ui.configbool('experimental', 'obshashrange', False)
+    if obsolete.isenabled(repo, obsolete.exchangeopt) and enabled:
+        caps = caps.split()
+        caps.append('_evoext_obshashrange_v0')
+        caps.sort()
+        caps = ' '.join(caps)
+    return caps
+
[e-mail]
+def obshashrange_extsetup(ui):
+    hgweb_mod.perms['evoext_obshashrange_v0'] = 'pull'
+
+    wireproto.commands['evoext_obshashrange_v0'] = (srv_obshashrange_v0, 'ranges')
+    ###
+    extensions.wrapfunction(wireproto, 'capabilities', _obshashrange_capabilities)
+    # wrap command content
+    oldcap, args = wireproto.commands['capabilities']
+
+    def newcap(repo, proto):
+        return _obshashrange_capabilities(oldcap, repo, proto)
+    wireproto.commands['capabilities'] = (newcap, args)
+
 #############################
 ### Tree Hash computation ###
 #############################