py3: have a bytes version of shlex.split()

py3: have a bytes version of shlex.split()

shlex.split() only accepts unicodes on Python 3. After this patch we will be
using pycompat.shlexsplit(). This patch also replaces existing occurences of
shlex.split with pycompat.shlexsplit.

--- a/hgext/extdiff.py	Fri Dec 23 16:26:40 2016 +0000
+++ b/hgext/extdiff.py	Sun Dec 25 03:06:55 2016 +0530
@@ -64,7 +64,6 @@
 
 import os
 import re
-import shlex
 import shutil
 import tempfile
 from mercurial.i18n import _
@@ -78,6 +77,7 @@
     commands,
     error,
     filemerge,
+    pycompat,
     scmutil,
     util,
 )
@@ -371,7 +371,7 @@
             if path:
                 # case "cmd = path opts"
                 cmdline = path
-                diffopts = len(shlex.split(cmdline)) > 1
+                diffopts = len(pycompat.shlexsplit(cmdline)) > 1
             else:
                 # case "cmd ="
                 path = util.findexe(cmd)

--- a/mercurial/commands.py	Fri Dec 23 16:26:40 2016 +0000
+++ b/mercurial/commands.py	Sun Dec 25 03:06:55 2016 +0530
@@ -11,7 +11,6 @@
 import errno
 import os
 import re
-import shlex
 import socket
 import string
 import sys
@@ -1981,7 +1980,7 @@
     editor = ui.geteditor()
     editor = util.expandpath(editor)
     fm.write('editor', _("checking commit editor... (%s)\n"), editor)
-    cmdpath = util.findexe(shlex.split(editor)[0])
+    cmdpath = util.findexe(pycompat.shlexsplit(editor)[0])
     fm.condwrite(not cmdpath and editor == 'vi', 'vinotfound',
                  _(" No commit editor set and can't find %s in PATH\n"
                    " (specify a commit editor in your configuration"

--- a/mercurial/dispatch.py	Fri Dec 23 16:26:40 2016 +0000
+++ b/mercurial/dispatch.py	Sun Dec 25 03:06:55 2016 +0530
@@ -14,7 +14,6 @@
 import os
 import pdb
 import re
-import shlex
 import signal
 import sys
 import time
@@ -279,7 +278,7 @@
         cmd = re.sub(r'\$(\d+|\$)', replacer, cmd)
         givenargs = [x for i, x in enumerate(givenargs)
                      if i not in nums]
-        args = shlex.split(cmd)
+        args = pycompat.shlexsplit(cmd)
     return args + givenargs
 
 def aliasinterpolate(name, args, cmd):
@@ -351,7 +350,7 @@
             return
 
         try:
-            args = shlex.split(self.definition)
+            args = pycompat.shlexsplit(self.definition)
         except ValueError as inst:
             self.badalias = (_("error in definition for alias '%s': %s")
                              % (self.name, inst))
@@ -461,7 +460,7 @@
         args = aliasargs(entry[0], args)
         defaults = ui.config("defaults", cmd)
         if defaults:
-            args = map(util.expandpath, shlex.split(defaults)) + args
+            args = map(util.expandpath, pycompat.shlexsplit(defaults)) + args
         c = list(entry[1])
     else:
         cmd = None

--- a/mercurial/pycompat.py	Fri Dec 23 16:26:40 2016 +0000
+++ b/mercurial/pycompat.py	Sun Dec 25 03:06:55 2016 +0530
@@ -12,6 +12,7 @@
 
 import getopt
 import os
+import shlex
 import sys
 
 ispy3 = (sys.version_info[0] >= 3)
@@ -122,6 +123,14 @@
         dic = dict((k.encode('latin-1'), v) for k, v in dic.iteritems())
         return dic
 
+    # shlex.split() accepts unicodes on Python 3. This function takes bytes
+    # argument, convert it into unicodes, pass into shlex.split(), convert the
+    # returned value to bytes and return that.
+    # TODO: handle shlex.shlex().
+    def shlexsplit(s):
+        ret = shlex.split(s.decode('latin-1'))
+        return [a.encode('latin-1') for a in ret]
+
 else:
     def sysstr(s):
         return s
@@ -162,6 +171,7 @@
     getcwd = os.getcwd
     osgetenv = os.getenv
     sysexecutable = sys.executable
+    shlexsplit = shlex.split
 
 stringio = io.StringIO
 empty = _queue.Empty