wireproto: crude support for version 2 HTTP peer
As part of implementing the server-side bits of the wire protocol
command handlers for version 2, we want a way to easily test those
commands. Currently, we use the "httprequest" action of `hg
debugwireproto`. But this requires explicitly specifying the HTTP
request headers, low-level frame details, and the data structure
to encode with CBOR. That's a lot of boilerplate and a lot of it can
change as the wire protocol evolves.
`hg debugwireproto` has a mechanism to issue commands via the peer
interface. That is *much* easier to use and we prefer to test with
that going forward.
This commit implements enough parts of the peer API to send basic
requests via the HTTP version 2 transport.
The peer code is super hacky. Again, the goal is to facilitate
server testing, not robustly implement a client. The client code
will receive love at a later time.
Differential Revision: https://phab.mercurial-scm.org/D3177
--- a/mercurial/debugcommands.py Mon Mar 26 15:34:52 2018 -0700
+++ b/mercurial/debugcommands.py Wed Mar 28 15:09:34 2018 -0700
@@ -2631,8 +2631,8 @@
``--peer`` can be used to bypass the handshake protocol and construct a
peer instance using the specified class type. Valid values are ``raw``,
- ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending raw data
- payloads and don't support higher-level command actions.
+ ``http2``, ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending
+ raw data payloads and don't support higher-level command actions.
``--noreadstderr`` can be used to disable automatic reading from stderr
of the peer (for SSH connections only). Disabling automatic reading of
@@ -2678,8 +2678,10 @@
command listkeys
namespace bookmarks
- Values are interpreted as Python b'' literals. This allows encoding
- special byte sequences via backslash escaping.
+ If the value begins with ``eval:``, it will be interpreted as a Python
+ literal expression. Otherwise values are interpreted as Python b'' literals.
+ This allows sending complex types and encoding special byte sequences via
+ backslash escaping.
The following arguments have special meaning:
@@ -2803,7 +2805,7 @@
if opts['localssh'] and not repo:
raise error.Abort(_('--localssh requires a repository'))
- if opts['peer'] and opts['peer'] not in ('raw', 'ssh1', 'ssh2'):
+ if opts['peer'] and opts['peer'] not in ('raw', 'http2', 'ssh1', 'ssh2'):
raise error.Abort(_('invalid value for --peer'),
hint=_('valid values are "raw", "ssh1", and "ssh2"'))
@@ -2877,18 +2879,20 @@
raise error.Abort(_('only http:// paths are currently supported'))
url, authinfo = u.authinfo()
- openerargs = {}
+ openerargs = {
+ r'useragent': b'Mercurial debugwireproto',
+ }
# Turn pipes/sockets into observers so we can log I/O.
if ui.verbose:
- openerargs = {
+ openerargs.update({
r'loggingfh': ui,
r'loggingname': b's',
r'loggingopts': {
r'logdata': True,
r'logdataapis': False,
},
- }
+ })
if ui.debugflag:
openerargs[r'loggingopts'][r'logdataapis'] = True
@@ -2901,7 +2905,10 @@
opener = urlmod.opener(ui, authinfo, **openerargs)
- if opts['peer'] == 'raw':
+ if opts['peer'] == 'http2':
+ ui.write(_('creating http peer for wire protocol version 2\n'))
+ peer = httppeer.httpv2peer(ui, path, opener)
+ elif opts['peer'] == 'raw':
ui.write(_('using raw connection to peer\n'))
peer = None
elif opts['peer']:
@@ -2951,7 +2958,12 @@
else:
key, value = fields
- args[key] = stringutil.unescapestr(value)
+ if value.startswith('eval:'):
+ value = stringutil.evalpythonliteral(value[5:])
+ else:
+ value = stringutil.unescapestr(value)
+
+ args[key] = value
if batchedcommands is not None:
batchedcommands.append((command, args))
--- a/mercurial/httppeer.py Mon Mar 26 15:34:52 2018 -0700
+++ b/mercurial/httppeer.py Wed Mar 28 15:09:34 2018 -0700
@@ -16,6 +16,9 @@
import tempfile
from .i18n import _
+from .thirdparty import (
+ cbor,
+)
from . import (
bundle2,
error,
@@ -25,6 +28,8 @@
url as urlmod,
util,
wireproto,
+ wireprotoframing,
+ wireprotoserver,
)
httplib = util.httplib
@@ -467,6 +472,95 @@
def _abort(self, exception):
raise exception
+# TODO implement interface for version 2 peers
+class httpv2peer(object):
+ def __init__(self, ui, repourl, opener):
+ self.ui = ui
+
+ if repourl.endswith('/'):
+ repourl = repourl[:-1]
+
+ self.url = repourl
+ self._opener = opener
+ # This is an its own attribute to facilitate extensions overriding
+ # the default type.
+ self._requestbuilder = urlreq.request
+
+ def close(self):
+ pass
+
+ # TODO require to be part of a batched primitive, use futures.
+ def _call(self, name, **args):
+ """Call a wire protocol command with arguments."""
+
+ # TODO permissions should come from capabilities results.
+ permission = wireproto.commandsv2[name].permission
+ if permission not in ('push', 'pull'):
+ raise error.ProgrammingError('unknown permission type: %s' %
+ permission)
+
+ permission = {
+ 'push': 'rw',
+ 'pull': 'ro',
+ }[permission]
+
+ url = '%s/api/%s/%s/%s' % (self.url, wireprotoserver.HTTPV2, permission,
+ name)
+
+ # TODO modify user-agent to reflect v2.
+ headers = {
+ r'Accept': wireprotoserver.FRAMINGTYPE,
+ r'Content-Type': wireprotoserver.FRAMINGTYPE,
+ }
+
+ # TODO this should be part of a generic peer for the frame-based
+ # protocol.
+ stream = wireprotoframing.stream(1)
+ frames = wireprotoframing.createcommandframes(stream, 1,
+ name, args)
+
+ body = b''.join(map(bytes, frames))
+ req = self._requestbuilder(pycompat.strurl(url), body, headers)
+ req.add_unredirected_header(r'Content-Length', r'%d' % len(body))
+
+ # TODO unify this code with httppeer.
+ try:
+ res = self._opener.open(req)
+ except urlerr.httperror as e:
+ if e.code == 401:
+ raise error.Abort(_('authorization failed'))
+
+ raise
+ except httplib.HTTPException as e:
+ self.ui.traceback()
+ raise IOError(None, e)
+
+ # TODO validate response type, wrap response to handle I/O errors.
+ # TODO more robust frame receiver.
+ results = []
+
+ while True:
+ frame = wireprotoframing.readframe(res)
+ if frame is None:
+ break
+
+ self.ui.note(_('received %r\n') % frame)
+
+ if frame.typeid == wireprotoframing.FRAME_TYPE_BYTES_RESPONSE:
+ if frame.flags & wireprotoframing.FLAG_BYTES_RESPONSE_CBOR:
+ payload = util.bytesio(frame.payload)
+
+ decoder = cbor.CBORDecoder(payload)
+ while payload.tell() + 1 < len(frame.payload):
+ results.append(decoder.decode())
+ else:
+ results.append(frame.payload)
+ else:
+ error.ProgrammingError('unhandled frame type: %d' %
+ frame.typeid)
+
+ return results
+
def makepeer(ui, path):
u = util.url(path)
if u.query or u.fragment:
--- a/tests/test-http-api-httpv2.t Mon Mar 26 15:34:52 2018 -0700
+++ b/tests/test-http-api-httpv2.t Wed Mar 28 15:09:34 2018 -0700
@@ -180,6 +180,36 @@
s> 0\r\n
s> \r\n
+ $ sendhttpv2peer << EOF
+ > command customreadonly
+ > EOF
+ creating http peer for wire protocol version 2
+ sending customreadonly command
+ s> POST /api/exp-http-v2-0001/ro/customreadonly HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0003\r\n
+ s> content-type: application/mercurial-exp-framing-0003\r\n
+ s> content-length: 29\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x15\x00\x00\x01\x00\x01\x01\x11\xa1DnameNcustomreadonly
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0003\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 25\r\n
+ s> \x1d\x00\x00\x01\x00\x02\x01B
+ s> customreadonly bytes response
+ s> \r\n
+ received frame(size=29; request=1; stream=2; streamflags=stream-begin; type=bytes-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ response: [b'customreadonly bytes response']
+
Request to read-write command fails because server is read-only by default
GET to read-write request yields 405
--- a/tests/test-http-protocol.t Mon Mar 26 15:34:52 2018 -0700
+++ b/tests/test-http-protocol.t Wed Mar 28 15:09:34 2018 -0700
@@ -179,7 +179,7 @@
s> Accept-Encoding: identity\r\n
s> accept: application/mercurial-0.1\r\n
s> host: $LOCALIP:$HGPORT\r\n (glob)
- s> user-agent: mercurial/proto-1.0 (Mercurial *)\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
s> \r\n
s> makefile('rb', None)
s> HTTP/1.1 200 Script output follows\r\n
@@ -197,7 +197,7 @@
s> x-hgproto-1: 0.1 0.2 comp=$USUAL_COMPRESSIONS$\r\n
s> accept: application/mercurial-0.1\r\n
s> host: $LOCALIP:$HGPORT\r\n (glob)
- s> user-agent: mercurial/proto-1.0 (Mercurial *)\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
s> \r\n
s> makefile('rb', None)
s> HTTP/1.1 200 Script output follows\r\n
--- a/tests/wireprotohelpers.sh Mon Mar 26 15:34:52 2018 -0700
+++ b/tests/wireprotohelpers.sh Wed Mar 28 15:09:34 2018 -0700
@@ -5,6 +5,10 @@
hg --verbose debugwireproto --peer raw http://$LOCALIP:$HGPORT/
}
+sendhttpv2peer() {
+ hg --verbose debugwireproto --peer http2 http://$LOCALIP:$HGPORT/
+}
+
cat > dummycommands.py << EOF
from mercurial import (
wireprototypes,