py3: import builtin wrappers automagically by code transformer
This should be less invasive than mucking builtins.
Since tokenize.untokenize() looks start/end positions of tokens, we calculates
them from the NEWLINE token of the future import.
--- a/mercurial/__init__.py Sun Aug 14 12:51:21 2016 +0900
+++ b/mercurial/__init__.py Tue Aug 16 12:35:15 2016 +0900
@@ -170,7 +170,7 @@
spec.loader = hgloader(spec.name, spec.origin)
return spec
- def replacetokens(tokens):
+ def replacetokens(tokens, fullname):
"""Transform a stream of tokens from raw to Python 3.
It is called by the custom module loading machinery to rewrite
@@ -184,6 +184,7 @@
REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
"""
+ futureimpline = False
for i, t in enumerate(tokens):
# Convert most string literals to byte literals. String literals
# in Python 2 are bytes. String literals in Python 3 are unicode.
@@ -217,6 +218,29 @@
t.line)
continue
+ # Insert compatibility imports at "from __future__ import" line.
+ # No '\n' should be added to preserve line numbers.
+ if (t.type == token.NAME and t.string == 'import' and
+ all(u.type == token.NAME for u in tokens[i - 2:i]) and
+ [u.string for u in tokens[i - 2:i]] == ['from', '__future__']):
+ futureimpline = True
+ if t.type == token.NEWLINE and futureimpline:
+ futureimpline = False
+ if fullname == 'mercurial.pycompat':
+ yield t
+ continue
+ r, c = t.start
+ l = (b'; from mercurial.pycompat import '
+ b'delattr, getattr, hasattr, setattr, xrange\n')
+ for u in tokenize.tokenize(io.BytesIO(l).readline):
+ if u.type in (tokenize.ENCODING, token.ENDMARKER):
+ continue
+ yield tokenize.TokenInfo(u.type, u.string,
+ (r, c + u.start[1]),
+ (r, c + u.end[1]),
+ '')
+ continue
+
try:
nexttoken = tokens[i + 1]
except IndexError:
@@ -279,7 +303,7 @@
# ``replacetoken`` or any mechanism that changes semantics of module
# loading is changed. Otherwise cached bytecode may get loaded without
# the new transformation mechanisms applied.
- BYTECODEHEADER = b'HG\x00\x01'
+ BYTECODEHEADER = b'HG\x00\x02'
class hgloader(importlib.machinery.SourceFileLoader):
"""Custom module loader that transforms source code.
@@ -338,7 +362,7 @@
"""Perform token transformation before compilation."""
buf = io.BytesIO(data)
tokens = tokenize.tokenize(buf.readline)
- data = tokenize.untokenize(replacetokens(list(tokens)))
+ data = tokenize.untokenize(replacetokens(list(tokens), self.name))
# Python's built-in importer strips frames from exceptions raised
# for this code. Unfortunately, that mechanism isn't extensible
# and our frame will be blamed for the import failure. There
--- a/mercurial/pycompat.py Sun Aug 14 12:51:21 2016 +0900
+++ b/mercurial/pycompat.py Tue Aug 16 12:35:15 2016 +0900
@@ -32,7 +32,6 @@
if sys.version_info[0] >= 3:
import builtins
import functools
- builtins.xrange = range
def _wrapattrfunc(f):
@functools.wraps(f)
@@ -42,10 +41,12 @@
return f(object, name, *args)
return w
+ # these wrappers are automagically imported by hgloader
delattr = _wrapattrfunc(builtins.delattr)
getattr = _wrapattrfunc(builtins.getattr)
hasattr = _wrapattrfunc(builtins.hasattr)
setattr = _wrapattrfunc(builtins.setattr)
+ xrange = builtins.range
stringio = io.StringIO
empty = _queue.Empty
--- a/tests/test-check-py3-compat.t Sun Aug 14 12:51:21 2016 +0900
+++ b/tests/test-check-py3-compat.t Tue Aug 16 12:35:15 2016 +0900
@@ -48,7 +48,7 @@
hgext/gpg.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
hgext/graphlog.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
hgext/hgk.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
- hgext/highlight/highlight.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
+ hgext/highlight/highlight.py: error importing module: <ImportError> No module named 'pygments' (line 13)
hgext/histedit.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
hgext/journal.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
hgext/keyword.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
@@ -122,52 +122,51 @@
mercurial/hook.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
mercurial/httpconnection.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
mercurial/httppeer.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
- mercurial/keepalive.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/localrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/lock.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/mail.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/manifest.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/match.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/mdiff.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/merge.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/minirst.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/namespaces.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/obsolete.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/patch.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/pathutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/peer.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/pure/mpatch.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/pure/parsers.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/pushkey.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/pvec.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
- mercurial/registrar.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/repair.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/repoview.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/revlog.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/revset.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/scmutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/scmwindows.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/similar.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/simplemerge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/sshpeer.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/sshserver.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/sslutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/statichttprepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/store.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/streamclone.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/subrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/tagmerge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/tags.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/templatefilters.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/templatekw.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/templater.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/transaction.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/ui.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/unionrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/url.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
- mercurial/verify.py: error importing: <TypeError> attribute name must be string, not 'bytes' (error at mdiff.py:*) (glob)
+ mercurial/keepalive.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/localrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/lock.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/mail.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/manifest.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/match.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/mdiff.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/merge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/minirst.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/namespaces.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/obsolete.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/patch.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/pathutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/peer.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/profiling.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/pushkey.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/pvec.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/registrar.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/repair.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/repoview.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/revlog.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/revset.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/scmutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/scmwindows.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/similar.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/simplemerge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/sshpeer.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/sshserver.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/sslutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/statichttprepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/store.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/streamclone.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/subrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/tagmerge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/tags.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/templatefilters.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/templatekw.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/templater.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/transaction.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/ui.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/unionrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/url.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+ mercurial/verify.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line 884)
mercurial/win32.py: error importing module: <ImportError> No module named 'msvcrt' (line *) (glob)
mercurial/windows.py: error importing module: <ImportError> No module named 'msvcrt' (line *) (glob)
- mercurial/wireproto.py: error importing module: <TypeError> a bytes-like object is required, not 'str' (line *) (glob)
+ mercurial/wireproto.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line 884)
#endif