url: support suppressing Accept header

url: support suppressing Accept header

Sending this header automatically could interfere with future
testing and client behavior. Let's add a knob to disable the
behavior.

We don't have a control for User-Agent because urllib will send
it if we don't set something. I don't feel like hacking into the
bowels of urllib to figure out how to suppress that. UA shouldn't
be used for anything meaningful. So it shouldn't pose any problems
beyond non-determinism (since the header has the Mercurial version in
it).

Differential Revision: https://phab.mercurial-scm.org/D2843

--- a/mercurial/debugcommands.py	Tue Mar 13 11:20:07 2018 -0700
+++ b/mercurial/debugcommands.py	Tue Mar 13 10:34:36 2018 -0700
@@ -2846,6 +2846,12 @@
         if ui.debugflag:
             openerargs[r'loggingopts'][r'logdataapis'] = True
 
+        # Don't send default headers when in raw mode. This allows us to
+        # bypass most of the behavior of our URL handling code so we can
+        # have near complete control over what's sent on the wire.
+        if opts['peer'] == 'raw':
+            openerargs[r'sendaccept'] = False
+
         opener = urlmod.opener(ui, authinfo, **openerargs)
 
         if opts['peer'] == 'raw':

--- a/mercurial/url.py	Tue Mar 13 11:20:07 2018 -0700
+++ b/mercurial/url.py	Tue Mar 13 10:34:36 2018 -0700
@@ -494,7 +494,7 @@
 handlerfuncs = []
 
 def opener(ui, authinfo=None, useragent=None, loggingfh=None,
-           loggingname=b's', loggingopts=None):
+           loggingname=b's', loggingopts=None, sendaccept=True):
     '''
     construct an opener suitable for urllib2
     authinfo will be added to the password manager
@@ -506,6 +506,9 @@
     ``loggingname`` denotes the name of the to print when logging.
     ``loggingopts`` is a dict of keyword arguments to pass to the constructed
     ``util.socketobserver`` instance.
+
+    ``sendaccept`` allows controlling whether the ``Accept`` request header
+    is sent. The header is sent by default.
     '''
     handlers = []
 
@@ -562,7 +565,9 @@
     # been sent on all requests since forever. We keep sending it for backwards
     # compatibility reasons. Modern versions of the wire protocol use
     # X-HgProto-<N> for advertising client support.
-    opener.addheaders.append((r'Accept', r'application/mercurial-0.1'))
+    if sendaccept:
+        opener.addheaders.append((r'Accept', r'application/mercurial-0.1'))
+
     return opener
 
 def open(ui, url_, data=None):

--- a/tests/test-http-protocol.t	Tue Mar 13 11:20:07 2018 -0700
+++ b/tests/test-http-protocol.t	Tue Mar 13 10:34:36 2018 -0700
@@ -215,15 +215,13 @@
 
   $ hg --verbose debugwireproto --peer raw http://$LOCALIP:$HGPORT << EOF
   > httprequest GET ?cmd=listkeys
-  >     accept: application/mercurial-0.1
-  >     user-agent: mercurial/proto-1.0 (Mercurial 42)
+  >     user-agent: test
   >     x-hgarg-1: namespace=namespaces
   > EOF
   using raw connection to peer
   s>     GET /?cmd=listkeys HTTP/1.1\r\n
   s>     Accept-Encoding: identity\r\n
-  s>     accept: application/mercurial-0.1\r\n
-  s>     user-agent: mercurial/proto-1.0 (Mercurial 42)\r\n
+  s>     user-agent: test\r\n
   s>     x-hgarg-1: namespace=namespaces\r\n
   s>     host: $LOCALIP:$HGPORT\r\n (glob)
   s>     \r\n