changeset 39645:a86d21e70b2b

exchangev2: start to implement pull with wire protocol v2 Wire protocol version 2 will take a substantially different approach to exchange than version 1 (at least as far as pulling is concerned). This commit establishes a new exchangev2 module for holding code related to exchange using wire protocol v2. I could have added things to the existing exchange module. But it is already quite big. And doing things inline isn't in question because the existing code is already littered with conditional code for various states of support for the existing wire protocol as it evolved over 10+ years. A new module gives us a chance to make a clean break. This approach does mean we'll end up writing some duplicate code. And there's a significant chance we'll miss functionality as code is ported. The plan is to eventually add #testcase's to existing tests so the new wire protocol is tested side-by-side with the existing one. This will hopefully tease out any features that weren't ported properly. But before we get there, we need to build up support for the new exchange methods. Our journey towards implementing a new exchange begins with pulling. And pulling begins with discovery. The discovery code added to exchangev2 is heavily drawn from the following functions: * exchange._pulldiscoverychangegroup * discovery.findcommonincoming For now, we build on top of existing discovery mechanisms. The new wire protocol should be capable of doing things more efficiently. But I'd rather defer on this problem. To foster the transition, we invent a fake capability on the HTTPv2 peer and have the main pull code in exchange.py call into exchangev2 when the new wire protocol is being used. Differential Revision: https://phab.mercurial-scm.org/D4480
author Gregory Szorc <gregory.szorc@gmail.com>
date Wed, 12 Sep 2018 09:58:23 -0700
parents a5de21c9e370
children 9c2c77c73f23
files mercurial/exchange.py mercurial/exchangev2.py mercurial/httppeer.py tests/test-wireproto-exchangev2.t tests/wireprotohelpers.sh
diffstat 5 files changed, 133 insertions(+), 12 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial/exchange.py	Tue Aug 21 15:33:11 2018 -0700
+++ b/mercurial/exchange.py	Wed Sep 12 09:58:23 2018 -0700
@@ -26,6 +26,7 @@
     changegroup,
     discovery,
     error,
+    exchangev2,
     lock as lockmod,
     logexchange,
     narrowspec,
@@ -1506,17 +1507,21 @@
 
     pullop.trmanager = transactionmanager(repo, 'pull', remote.url())
     with repo.wlock(), repo.lock(), pullop.trmanager:
-        # This should ideally be in _pullbundle2(). However, it needs to run
-        # before discovery to avoid extra work.
-        _maybeapplyclonebundle(pullop)
-        streamclone.maybeperformlegacystreamclone(pullop)
-        _pulldiscovery(pullop)
-        if pullop.canusebundle2:
-            _fullpullbundle2(repo, pullop)
-        _pullchangeset(pullop)
-        _pullphase(pullop)
-        _pullbookmarks(pullop)
-        _pullobsolete(pullop)
+        # Use the modern wire protocol, if available.
+        if remote.capable('exchangev2'):
+            exchangev2.pull(pullop)
+        else:
+            # This should ideally be in _pullbundle2(). However, it needs to run
+            # before discovery to avoid extra work.
+            _maybeapplyclonebundle(pullop)
+            streamclone.maybeperformlegacystreamclone(pullop)
+            _pulldiscovery(pullop)
+            if pullop.canusebundle2:
+                _fullpullbundle2(repo, pullop)
+            _pullchangeset(pullop)
+            _pullphase(pullop)
+            _pullbookmarks(pullop)
+            _pullobsolete(pullop)
 
     # storing remotenames
     if repo.ui.configbool('experimental', 'remotenames'):
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/exchangev2.py	Wed Sep 12 09:58:23 2018 -0700
@@ -0,0 +1,55 @@
+# exchangev2.py - repository exchange for wire protocol version 2
+#
+# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+from .node import (
+    nullid,
+)
+from . import (
+    setdiscovery,
+)
+
+def pull(pullop):
+    """Pull using wire protocol version 2."""
+    repo = pullop.repo
+    remote = pullop.remote
+
+    # Figure out what needs to be fetched.
+    common, fetch, remoteheads = _pullchangesetdiscovery(
+        repo, remote, pullop.heads, abortwhenunrelated=pullop.force)
+
+def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
+    """Determine which changesets need to be pulled."""
+
+    if heads:
+        knownnode = repo.changelog.hasnode
+        if all(knownnode(head) for head in heads):
+            return heads, False, heads
+
+    # TODO wire protocol version 2 is capable of more efficient discovery
+    # than setdiscovery. Consider implementing something better.
+    common, fetch, remoteheads = setdiscovery.findcommonheads(
+        repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated)
+
+    common = set(common)
+    remoteheads = set(remoteheads)
+
+    # If a remote head is filtered locally, put it back in the common set.
+    # See the comment in exchange._pulldiscoverychangegroup() for more.
+
+    if fetch and remoteheads:
+        nodemap = repo.unfiltered().changelog.nodemap
+
+        common |= {head for head in remoteheads if head in nodemap}
+
+        if set(remoteheads).issubset(common):
+            fetch = []
+
+    common.discard(nullid)
+
+    return common, fetch, remoteheads
--- a/mercurial/httppeer.py	Tue Aug 21 15:33:11 2018 -0700
+++ b/mercurial/httppeer.py	Wed Sep 12 09:58:23 2018 -0700
@@ -802,7 +802,8 @@
             return True
 
         # Other concepts.
-        if name in ('bundle2',):
+        # TODO remove exchangev2 once we have a command implemented.
+        if name in ('bundle2', 'exchangev2'):
             return True
 
         # Alias command-* to presence of command of that name.
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-wireproto-exchangev2.t	Wed Sep 12 09:58:23 2018 -0700
@@ -0,0 +1,53 @@
+Tests for wire protocol version 2 exchange.
+Tests in this file should be folded into existing tests once protocol
+v2 has enough features that it can be enabled via #testcase in existing
+tests.
+
+  $ . $TESTDIR/wireprotohelpers.sh
+  $ enablehttpv2client
+
+  $ hg init server-simple
+  $ enablehttpv2 server-simple
+  $ cd server-simple
+  $ cat >> .hg/hgrc << EOF
+  > [phases]
+  > publish = false
+  > EOF
+  $ echo a0 > a
+  $ echo b0 > b
+  $ hg -q commit -A -m 'commit 0'
+
+  $ echo a1 > a
+  $ hg commit -m 'commit 1'
+  $ hg phase --public -r .
+  $ echo a2 > a
+  $ hg commit -m 'commit 2'
+
+  $ hg -q up -r 0
+  $ echo b1 > b
+  $ hg -q commit -m 'head 2 commit 1'
+  $ echo b2 > b
+  $ hg -q commit -m 'head 2 commit 2'
+
+  $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+  $ cat hg.pid > $DAEMON_PIDS
+
+  $ cd ..
+
+Test basic clone
+
+  $ hg --debug clone -U http://localhost:$HGPORT client-simple
+  using http://localhost:$HGPORT/
+  sending capabilities command
+  query 1; heads
+  sending 2 commands
+  sending command heads: {}
+  sending command known: {
+    'nodes': []
+  }
+  received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation)
+  received frame(size=43; request=1; stream=2; streamflags=; type=command-response; flags=continuation)
+  received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
+  received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation)
+  received frame(size=1; request=3; stream=2; streamflags=; type=command-response; flags=continuation)
+  received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
--- a/tests/wireprotohelpers.sh	Tue Aug 21 15:33:11 2018 -0700
+++ b/tests/wireprotohelpers.sh	Wed Sep 12 09:58:23 2018 -0700
@@ -56,3 +56,10 @@
 web.api.http-v2 = true
 EOF
 }
+
+enablehttpv2client() {
+  cat >> $HGRCPATH << EOF
+[experimental]
+httppeer.advertise-v2 = true
+EOF
+}