changeset 2054:f9d65d24b9f9

discovery: split discovery related code in 'obsdiscovery'

More code splitting for more clarity.

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgext3rd/evolve/obsdiscovery.py	Tue Mar 07 14:29:43 2017 +0100
@@ -0,0 +1,204 @@
+# Code dedicated to the discovery of obsolescence marker "over the wire"
+#
+# Copyright 2017 Pierre-Yves David <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+try:
+    import StringIO as io
+    StringIO = io.StringIO
+except ImportError:
+    import io
+    StringIO = io.StringIO
+
+from mercurial import (
+    error,
+    exchange,
+    localrepo,
+    node,
+    obsolete,
+    wireproto,
+)
+from mercurial.i18n import _
+
+from . import (
+    exthelper,
+    serveronly,
+    utility,
+)
+
+eh = exthelper.exthelper()
+obsexcmsg = utility.obsexcmsg
+
+
+from mercurial import dagutil
+from mercurial import setdiscovery
+
[e-mail](localrepo.localpeer, 'evoext_obshash')
+def local_obshash(peer, nodes):
+    return serveronly._obshash(peer._repo, nodes)
+
[e-mail](localrepo.localpeer, 'evoext_obshash1')
+def local_obshash1(peer, nodes):
+    return serveronly._obshash(peer._repo, nodes, version=1)
+
[e-mail](wireproto.wirepeer, 'evoext_obshash')
+def peer_obshash(self, nodes):
+    d = self._call("evoext_obshash", nodes=wireproto.encodelist(nodes))
+    try:
+        return wireproto.decodelist(d)
+    except ValueError:
+        self._abort(error.ResponseError(_("unexpected response:"), d))
+
[e-mail](wireproto.wirepeer, 'evoext_obshash1')
+def peer_obshash1(self, nodes):
+    d = self._call("evoext_obshash1", nodes=wireproto.encodelist(nodes))
+    try:
+        return wireproto.decodelist(d)
+    except ValueError:
+        self._abort(error.ResponseError(_("unexpected response:"), d))
+
+def findcommonobsmarkers(ui, local, remote, probeset,
+                         initialsamplesize=100,
+                         fullsamplesize=200):
+    # from discovery
+    roundtrips = 0
+    cl = local.changelog
+    dag = dagutil.revlogdag(cl)
+    missing = set()
+    common = set()
+    undecided = set(probeset)
+    totalnb = len(undecided)
+    ui.progress(_("comparing with other"), 0, total=totalnb)
+    _takefullsample = setdiscovery._takefullsample
+    if remote.capable('_evoext_obshash_1'):
+        getremotehash = remote.evoext_obshash1
+        localhash = serveronly._obsrelsethashtreefm1(local)
+    else:
+        getremotehash = remote.evoext_obshash
+        localhash = serveronly._obsrelsethashtreefm0(local)
+
+    while undecided:
+
+        ui.note(_("sampling from both directions\n"))
+        if len(undecided) < fullsamplesize:
+            sample = set(undecided)
+        else:
+            sample = _takefullsample(dag, undecided, size=fullsamplesize)
+
+        roundtrips += 1
+        ui.progress(_("comparing with other"), totalnb - len(undecided),
+                    total=totalnb)
+        ui.debug("query %i; still undecided: %i, sample size is: %i\n"
+                 % (roundtrips, len(undecided), len(sample)))
+        # indices between sample and externalized version must match
+        sample = list(sample)
+        remotehash = getremotehash(dag.externalizeall(sample))
+
+        yesno = [localhash[ix][1] == remotehash[si]
+                 for si, ix in enumerate(sample)]
+
+        commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
+        common.update(dag.ancestorset(commoninsample, common))
+
+        missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
+        missing.update(dag.descendantset(missinginsample, missing))
+
+        undecided.difference_update(missing)
+        undecided.difference_update(common)
+
+    ui.progress(_("comparing with other"), None)
+    result = dag.headsetofconnecteds(common)
+    ui.debug("%d total queries\n" % roundtrips)
+
+    if not result:
+        return set([node.nullid])
+    return dag.externalizeall(result)
+
[e-mail](exchange, '_pushdiscoveryobsmarkers')
+def _pushdiscoveryobsmarkers(orig, pushop):
+    if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt)
+        and pushop.repo.obsstore
+        and 'obsolete' in pushop.remote.listkeys('namespaces')):
+        repo = pushop.repo
+        obsexcmsg(repo.ui, "computing relevant nodes\n")
+        revs = list(repo.revs('::%ln', pushop.futureheads))
+        unfi = repo.unfiltered()
+        cl = unfi.changelog
+        if not pushop.remote.capable('_evoext_obshash_0'):
+            # do not trust core yet
+            # return orig(pushop)
+            nodes = [cl.node(r) for r in revs]
+            if nodes:
+                obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
+                                   % len(nodes))
+                pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
+            else:
+                obsexcmsg(repo.ui, "markers already in sync\n")
+                pushop.outobsmarkers = []
+                pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
+            return
+
+        common = []
+        obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
+                           % len(revs))
+        commonrevs = list(unfi.revs('::%ln', pushop.outgoing.commonheads))
+        common = findcommonobsmarkers(pushop.ui, unfi, pushop.remote,
+                                      commonrevs)
+
+        revs = list(unfi.revs('%ld - (::%ln)', revs, common))
+        nodes = [cl.node(r) for r in revs]
+        if nodes:
+            obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
+                               % len(nodes))
+            pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
+        else:
+            obsexcmsg(repo.ui, "markers already in sync\n")
+            pushop.outobsmarkers = []
+
[e-mail]
+def _installobsmarkersdiscovery(ui):
+    olddisco = exchange.pushdiscoverymapping['obsmarker']
+
+    def newdisco(pushop):
+        _pushdiscoveryobsmarkers(olddisco, pushop)
+    exchange.pushdiscoverymapping['obsmarker'] = newdisco
+
+def _buildpullobsmarkersboundaries(pullop):
+    """small funtion returning the argument for pull markers call
+    may to contains 'heads' and 'common'. skip the key for None.
+
+    Its a separed functio to play around with strategy for that."""
+    repo = pullop.repo
+    remote = pullop.remote
+    unfi = repo.unfiltered()
+    revs = unfi.revs('::(%ln - null)', pullop.common)
+    common = [node.nullid]
+    if remote.capable('_evoext_obshash_0'):
+        obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
+                           % len(revs))
+        common = findcommonobsmarkers(repo.ui, repo, remote, revs)
+    return {'heads': pullop.pulledsubset, 'common': common}
+
[e-mail](
+    'debugobsrelsethashtree',
+    [('', 'v0', None, 'hash on marker format "0"'),
+     ('', 'v1', None, 'hash on marker format "1" (default)')], _(''))
+def debugobsrelsethashtree(ui, repo, v0=False, v1=False):
+    """display Obsolete markers, Relevant Set, Hash Tree
+    changeset-node obsrelsethashtree-node
+
+    It computed form the "orsht" of its parent and markers
+    relevant to the changeset itself."""
+    if v0 and v1:
+        raise error.Abort('cannot only specify one format')
+    elif v0:
+        treefunc = serveronly._obsrelsethashtreefm0
+    else:
+        treefunc = serveronly._obsrelsethashtreefm1
+
+    for chg, obs in treefunc(repo):
+        ui.status('%s %s\n' % (node.hex(chg), node.hex(obs)))

--- a/hgext3rd/evolve/obsexchange.py	Tue Mar 07 14:19:12 2017 +0100
+++ b/hgext3rd/evolve/obsexchange.py	Tue Mar 07 14:29:43 2017 +0100
@@ -33,152 +33,13 @@
     exthelper,
     serveronly,
     utility,
+    obsdiscovery,
 )
 
 eh = exthelper.exthelper()
+eh.merge(obsdiscovery.eh)
 obsexcmsg = utility.obsexcmsg
-
-def obsexcprg(ui, *args, **kwargs):
-    topic = 'obsmarkers exchange'
-    if ui.configbool('experimental', 'verbose-obsolescence-exchange', False):
-        topic = 'OBSEXC'
-    ui.progress(topic, *args, **kwargs)
-
[e-mail](exchange, '_pushdiscoveryobsmarkers')
-def _pushdiscoveryobsmarkers(orig, pushop):
-    if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt)
-        and pushop.repo.obsstore
-        and 'obsolete' in pushop.remote.listkeys('namespaces')):
-        repo = pushop.repo
-        obsexcmsg(repo.ui, "computing relevant nodes\n")
-        revs = list(repo.revs('::%ln', pushop.futureheads))
-        unfi = repo.unfiltered()
-        cl = unfi.changelog
-        if not pushop.remote.capable('_evoext_obshash_0'):
-            # do not trust core yet
-            # return orig(pushop)
-            nodes = [cl.node(r) for r in revs]
-            if nodes:
-                obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
-                                   % len(nodes))
-                pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
-            else:
-                obsexcmsg(repo.ui, "markers already in sync\n")
-                pushop.outobsmarkers = []
-                pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
-            return
-
-        common = []
-        obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
-                           % len(revs))
-        commonrevs = list(unfi.revs('::%ln', pushop.outgoing.commonheads))
-        common = findcommonobsmarkers(pushop.ui, unfi, pushop.remote,
-                                      commonrevs)
-
-        revs = list(unfi.revs('%ld - (::%ln)', revs, common))
-        nodes = [cl.node(r) for r in revs]
-        if nodes:
-            obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
-                               % len(nodes))
-            pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
-        else:
-            obsexcmsg(repo.ui, "markers already in sync\n")
-            pushop.outobsmarkers = []
-
[e-mail]
-def _installobsmarkersdiscovery(ui):
-    olddisco = exchange.pushdiscoverymapping['obsmarker']
-
-    def newdisco(pushop):
-        _pushdiscoveryobsmarkers(olddisco, pushop)
-    exchange.pushdiscoverymapping['obsmarker'] = newdisco
-
-### Set discovery START
-
-from mercurial import dagutil
-from mercurial import setdiscovery
-
[e-mail](localrepo.localpeer, 'evoext_obshash')
-def local_obshash(peer, nodes):
-    return serveronly._obshash(peer._repo, nodes)
-
[e-mail](localrepo.localpeer, 'evoext_obshash1')
-def local_obshash1(peer, nodes):
-    return serveronly._obshash(peer._repo, nodes, version=1)
-
[e-mail](wireproto.wirepeer, 'evoext_obshash')
-def peer_obshash(self, nodes):
-    d = self._call("evoext_obshash", nodes=wireproto.encodelist(nodes))
-    try:
-        return wireproto.decodelist(d)
-    except ValueError:
-        self._abort(error.ResponseError(_("unexpected response:"), d))
-
[e-mail](wireproto.wirepeer, 'evoext_obshash1')
-def peer_obshash1(self, nodes):
-    d = self._call("evoext_obshash1", nodes=wireproto.encodelist(nodes))
-    try:
-        return wireproto.decodelist(d)
-    except ValueError:
-        self._abort(error.ResponseError(_("unexpected response:"), d))
-
-def findcommonobsmarkers(ui, local, remote, probeset,
-                         initialsamplesize=100,
-                         fullsamplesize=200):
-    # from discovery
-    roundtrips = 0
-    cl = local.changelog
-    dag = dagutil.revlogdag(cl)
-    missing = set()
-    common = set()
-    undecided = set(probeset)
-    totalnb = len(undecided)
-    ui.progress(_("comparing with other"), 0, total=totalnb)
-    _takefullsample = setdiscovery._takefullsample
-    if remote.capable('_evoext_obshash_1'):
-        getremotehash = remote.evoext_obshash1
-        localhash = serveronly._obsrelsethashtreefm1(local)
-    else:
-        getremotehash = remote.evoext_obshash
-        localhash = serveronly._obsrelsethashtreefm0(local)
-
-    while undecided:
-
-        ui.note(_("sampling from both directions\n"))
-        if len(undecided) < fullsamplesize:
-            sample = set(undecided)
-        else:
-            sample = _takefullsample(dag, undecided, size=fullsamplesize)
-
-        roundtrips += 1
-        ui.progress(_("comparing with other"), totalnb - len(undecided),
-                    total=totalnb)
-        ui.debug("query %i; still undecided: %i, sample size is: %i\n"
-                 % (roundtrips, len(undecided), len(sample)))
-        # indices between sample and externalized version must match
-        sample = list(sample)
-        remotehash = getremotehash(dag.externalizeall(sample))
-
-        yesno = [localhash[ix][1] == remotehash[si]
-                 for si, ix in enumerate(sample)]
-
-        commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
-        common.update(dag.ancestorset(commoninsample, common))
-
-        missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
-        missing.update(dag.descendantset(missinginsample, missing))
-
-        undecided.difference_update(missing)
-        undecided.difference_update(common)
-
-    ui.progress(_("comparing with other"), None)
-    result = dag.headsetofconnecteds(common)
-    ui.debug("%d total queries\n" % roundtrips)
-
-    if not result:
-        return set([node.nullid])
-    return dag.externalizeall(result)
-
+obsexcprg = utility.obsexcprg
 
 _pushkeyescape = getattr(obsolete, '_pushkeyescape', None)
 
@@ -293,22 +154,6 @@
     data = obsfile.read()
     serveronly._pushobsmarkers(peer._repo, data)
 
-def _buildpullobsmarkersboundaries(pullop):
-    """small funtion returning the argument for pull markers call
-    may to contains 'heads' and 'common'. skip the key for None.
-
-    Its a separed functio to play around with strategy for that."""
-    repo = pullop.repo
-    remote = pullop.remote
-    unfi = repo.unfiltered()
-    revs = unfi.revs('::(%ln - null)', pullop.common)
-    common = [node.nullid]
-    if remote.capable('_evoext_obshash_0'):
-        obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
-                           % len(revs))
-        common = findcommonobsmarkers(repo.ui, repo, remote, revs)
-    return {'heads': pullop.pulledsubset, 'common': common}
-
 @eh.uisetup
 def addgetbundleargs(self):
     wireproto.gboptsmap['evo_obscommon'] = 'nodes'
@@ -318,7 +163,7 @@
     ret = orig(pullop, kwargs)
     if ('obsmarkers' in kwargs and
         pullop.remote.capable('_evoext_getbundle_obscommon')):
-        boundaries = _buildpullobsmarkersboundaries(pullop)
+        boundaries = obsdiscovery._buildpullobsmarkersboundaries(pullop)
         common = boundaries['common']
         if common != [node.nullid]:
             kwargs['evo_obscommon'] = common
@@ -339,7 +184,7 @@
         return None # remote opted out of obsolescence marker exchange
     tr = None
     ui = pullop.repo.ui
-    boundaries = _buildpullobsmarkersboundaries(pullop)
+    boundaries = obsdiscovery._buildpullobsmarkersboundaries(pullop)
     if not set(boundaries['heads']) - set(boundaries['common']):
         obsexcmsg(ui, "nothing to pull\n")
         return None
@@ -393,25 +238,4 @@
 def local_pullobsmarkers(self, heads=None, common=None):
     return serveronly._getobsmarkersstream(self._repo, heads=heads,
                                            common=common)
-
[e-mail](
-    'debugobsrelsethashtree',
-    [('', 'v0', None, 'hash on marker format "0"'),
-     ('', 'v1', None, 'hash on marker format "1" (default)')], _(''))
-def debugobsrelsethashtree(ui, repo, v0=False, v1=False):
-    """display Obsolete markers, Relevant Set, Hash Tree
-    changeset-node obsrelsethashtree-node
-
-    It computed form the "orsht" of its parent and markers
-    relevant to the changeset itself."""
-    if v0 and v1:
-        raise error.Abort('cannot only specify one format')
-    elif v0:
-        treefunc = serveronly._obsrelsethashtreefm0
-    else:
-        treefunc = serveronly._obsrelsethashtreefm1
-
-    for chg, obs in treefunc(repo):
-        ui.status('%s %s\n' % (node.hex(chg), node.hex(obs)))
-
 _bestformat = max(obsolete.formats.keys())

--- a/hgext3rd/evolve/utility.py	Tue Mar 07 14:19:12 2017 +0100
+++ b/hgext3rd/evolve/utility.py	Tue Mar 07 14:29:43 2017 +0100
@@ -12,3 +12,9 @@
         message = 'OBSEXC: ' + message
     if important or verbose:
         ui.status(message)
+
+def obsexcprg(ui, *args, **kwargs):
+    topic = 'obsmarkers exchange'
+    if ui.configbool('experimental', 'verbose-obsolescence-exchange', False):
+        topic = 'OBSEXC'
+    ui.progress(topic, *args, **kwargs)