Mercurial > hg
changeset 37483:61e405fb6372
wireproto: crude support for version 2 HTTP peer
As part of implementing the server-side bits of the wire protocol
command handlers for version 2, we want a way to easily test those
commands. Currently, we use the "httprequest" action of `hg
debugwireproto`. But this requires explicitly specifying the HTTP
request headers, low-level frame details, and the data structure
to encode with CBOR. That's a lot of boilerplate and a lot of it can
change as the wire protocol evolves.
`hg debugwireproto` has a mechanism to issue commands via the peer
interface. That is *much* easier to use and we prefer to test with
that going forward.
This commit implements enough parts of the peer API to send basic
requests via the HTTP version 2 transport.
The peer code is super hacky. Again, the goal is to facilitate
server testing, not robustly implement a client. The client code
will receive love at a later time.
Differential Revision: https://phab.mercurial-scm.org/D3177
| author | Gregory Szorc <gregory.szorc@gmail.com> |
|---|---|
| date | Wed, 28 Mar 2018 15:09:34 -0700 |
| parents | fa9faf58959d |
| children | c22fd3c4c23e |
| files | mercurial/debugcommands.py mercurial/httppeer.py tests/test-http-api-httpv2.t tests/test-http-protocol.t tests/wireprotohelpers.sh |
| diffstat | 5 files changed, 152 insertions(+), 12 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/debugcommands.py Mon Mar 26 15:34:52 2018 -0700 +++ b/mercurial/debugcommands.py Wed Mar 28 15:09:34 2018 -0700 @@ -2631,8 +2631,8 @@ ``--peer`` can be used to bypass the handshake protocol and construct a peer instance using the specified class type. Valid values are ``raw``, - ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending raw data - payloads and don't support higher-level command actions. + ``http2``, ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending + raw data payloads and don't support higher-level command actions. ``--noreadstderr`` can be used to disable automatic reading from stderr of the peer (for SSH connections only). Disabling automatic reading of @@ -2678,8 +2678,10 @@ command listkeys namespace bookmarks - Values are interpreted as Python b'' literals. This allows encoding - special byte sequences via backslash escaping. + If the value begins with ``eval:``, it will be interpreted as a Python + literal expression. Otherwise values are interpreted as Python b'' literals. + This allows sending complex types and encoding special byte sequences via + backslash escaping. The following arguments have special meaning: @@ -2803,7 +2805,7 @@ if opts['localssh'] and not repo: raise error.Abort(_('--localssh requires a repository')) - if opts['peer'] and opts['peer'] not in ('raw', 'ssh1', 'ssh2'): + if opts['peer'] and opts['peer'] not in ('raw', 'http2', 'ssh1', 'ssh2'): raise error.Abort(_('invalid value for --peer'), hint=_('valid values are "raw", "ssh1", and "ssh2"')) @@ -2877,18 +2879,20 @@ raise error.Abort(_('only http:// paths are currently supported')) url, authinfo = u.authinfo() - openerargs = {} + openerargs = { + r'useragent': b'Mercurial debugwireproto', + } # Turn pipes/sockets into observers so we can log I/O. if ui.verbose: - openerargs = { + openerargs.update({ r'loggingfh': ui, r'loggingname': b's', r'loggingopts': { r'logdata': True, r'logdataapis': False, }, - } + }) if ui.debugflag: openerargs[r'loggingopts'][r'logdataapis'] = True @@ -2901,7 +2905,10 @@ opener = urlmod.opener(ui, authinfo, **openerargs) - if opts['peer'] == 'raw': + if opts['peer'] == 'http2': + ui.write(_('creating http peer for wire protocol version 2\n')) + peer = httppeer.httpv2peer(ui, path, opener) + elif opts['peer'] == 'raw': ui.write(_('using raw connection to peer\n')) peer = None elif opts['peer']: @@ -2951,7 +2958,12 @@ else: key, value = fields - args[key] = stringutil.unescapestr(value) + if value.startswith('eval:'): + value = stringutil.evalpythonliteral(value[5:]) + else: + value = stringutil.unescapestr(value) + + args[key] = value if batchedcommands is not None: batchedcommands.append((command, args))
--- a/mercurial/httppeer.py Mon Mar 26 15:34:52 2018 -0700 +++ b/mercurial/httppeer.py Wed Mar 28 15:09:34 2018 -0700 @@ -16,6 +16,9 @@ import tempfile from .i18n import _ +from .thirdparty import ( + cbor, +) from . import ( bundle2, error, @@ -25,6 +28,8 @@ url as urlmod, util, wireproto, + wireprotoframing, + wireprotoserver, ) httplib = util.httplib @@ -467,6 +472,95 @@ def _abort(self, exception): raise exception +# TODO implement interface for version 2 peers +class httpv2peer(object): + def __init__(self, ui, repourl, opener): + self.ui = ui + + if repourl.endswith('/'): + repourl = repourl[:-1] + + self.url = repourl + self._opener = opener + # This is an its own attribute to facilitate extensions overriding + # the default type. + self._requestbuilder = urlreq.request + + def close(self): + pass + + # TODO require to be part of a batched primitive, use futures. + def _call(self, name, **args): + """Call a wire protocol command with arguments.""" + + # TODO permissions should come from capabilities results. + permission = wireproto.commandsv2[name].permission + if permission not in ('push', 'pull'): + raise error.ProgrammingError('unknown permission type: %s' % + permission) + + permission = { + 'push': 'rw', + 'pull': 'ro', + }[permission] + + url = '%s/api/%s/%s/%s' % (self.url, wireprotoserver.HTTPV2, permission, + name) + + # TODO modify user-agent to reflect v2. + headers = { + r'Accept': wireprotoserver.FRAMINGTYPE, + r'Content-Type': wireprotoserver.FRAMINGTYPE, + } + + # TODO this should be part of a generic peer for the frame-based + # protocol. + stream = wireprotoframing.stream(1) + frames = wireprotoframing.createcommandframes(stream, 1, + name, args) + + body = b''.join(map(bytes, frames)) + req = self._requestbuilder(pycompat.strurl(url), body, headers) + req.add_unredirected_header(r'Content-Length', r'%d' % len(body)) + + # TODO unify this code with httppeer. + try: + res = self._opener.open(req) + except urlerr.httperror as e: + if e.code == 401: + raise error.Abort(_('authorization failed')) + + raise + except httplib.HTTPException as e: + self.ui.traceback() + raise IOError(None, e) + + # TODO validate response type, wrap response to handle I/O errors. + # TODO more robust frame receiver. + results = [] + + while True: + frame = wireprotoframing.readframe(res) + if frame is None: + break + + self.ui.note(_('received %r\n') % frame) + + if frame.typeid == wireprotoframing.FRAME_TYPE_BYTES_RESPONSE: + if frame.flags & wireprotoframing.FLAG_BYTES_RESPONSE_CBOR: + payload = util.bytesio(frame.payload) + + decoder = cbor.CBORDecoder(payload) + while payload.tell() + 1 < len(frame.payload): + results.append(decoder.decode()) + else: + results.append(frame.payload) + else: + error.ProgrammingError('unhandled frame type: %d' % + frame.typeid) + + return results + def makepeer(ui, path): u = util.url(path) if u.query or u.fragment:
--- a/tests/test-http-api-httpv2.t Mon Mar 26 15:34:52 2018 -0700 +++ b/tests/test-http-api-httpv2.t Wed Mar 28 15:09:34 2018 -0700 @@ -180,6 +180,36 @@ s> 0\r\n s> \r\n + $ sendhttpv2peer << EOF + > command customreadonly + > EOF + creating http peer for wire protocol version 2 + sending customreadonly command + s> POST /api/exp-http-v2-0001/ro/customreadonly HTTP/1.1\r\n + s> Accept-Encoding: identity\r\n + s> accept: application/mercurial-exp-framing-0003\r\n + s> content-type: application/mercurial-exp-framing-0003\r\n + s> content-length: 29\r\n + s> host: $LOCALIP:$HGPORT\r\n (glob) + s> user-agent: Mercurial debugwireproto\r\n + s> \r\n + s> \x15\x00\x00\x01\x00\x01\x01\x11\xa1DnameNcustomreadonly + s> makefile('rb', None) + s> HTTP/1.1 200 OK\r\n + s> Server: testing stub value\r\n + s> Date: $HTTP_DATE$\r\n + s> Content-Type: application/mercurial-exp-framing-0003\r\n + s> Transfer-Encoding: chunked\r\n + s> \r\n + s> 25\r\n + s> \x1d\x00\x00\x01\x00\x02\x01B + s> customreadonly bytes response + s> \r\n + received frame(size=29; request=1; stream=2; streamflags=stream-begin; type=bytes-response; flags=eos) + s> 0\r\n + s> \r\n + response: [b'customreadonly bytes response'] + Request to read-write command fails because server is read-only by default GET to read-write request yields 405
--- a/tests/test-http-protocol.t Mon Mar 26 15:34:52 2018 -0700 +++ b/tests/test-http-protocol.t Wed Mar 28 15:09:34 2018 -0700 @@ -179,7 +179,7 @@ s> Accept-Encoding: identity\r\n s> accept: application/mercurial-0.1\r\n s> host: $LOCALIP:$HGPORT\r\n (glob) - s> user-agent: mercurial/proto-1.0 (Mercurial *)\r\n (glob) + s> user-agent: Mercurial debugwireproto\r\n s> \r\n s> makefile('rb', None) s> HTTP/1.1 200 Script output follows\r\n @@ -197,7 +197,7 @@ s> x-hgproto-1: 0.1 0.2 comp=$USUAL_COMPRESSIONS$\r\n s> accept: application/mercurial-0.1\r\n s> host: $LOCALIP:$HGPORT\r\n (glob) - s> user-agent: mercurial/proto-1.0 (Mercurial *)\r\n (glob) + s> user-agent: Mercurial debugwireproto\r\n s> \r\n s> makefile('rb', None) s> HTTP/1.1 200 Script output follows\r\n
--- a/tests/wireprotohelpers.sh Mon Mar 26 15:34:52 2018 -0700 +++ b/tests/wireprotohelpers.sh Wed Mar 28 15:09:34 2018 -0700 @@ -5,6 +5,10 @@ hg --verbose debugwireproto --peer raw http://$LOCALIP:$HGPORT/ } +sendhttpv2peer() { + hg --verbose debugwireproto --peer http2 http://$LOCALIP:$HGPORT/ +} + cat > dummycommands.py << EOF from mercurial import ( wireprototypes,