# HG changeset patch # User Yuya Nishihara # Date 1471318515 -32400 # Node ID 178c89e8519a6111072a4331601efb34f63de59e # Parent 45fa8de47a0f033428b29d4bb8a6291fb16019da py3: import builtin wrappers automagically by code transformer This should be less invasive than mucking builtins. Since tokenize.untokenize() looks start/end positions of tokens, we calculates them from the NEWLINE token of the future import. diff -r 45fa8de47a0f -r 178c89e8519a mercurial/__init__.py --- a/mercurial/__init__.py Sun Aug 14 12:51:21 2016 +0900 +++ b/mercurial/__init__.py Tue Aug 16 12:35:15 2016 +0900 @@ -170,7 +170,7 @@ spec.loader = hgloader(spec.name, spec.origin) return spec - def replacetokens(tokens): + def replacetokens(tokens, fullname): """Transform a stream of tokens from raw to Python 3. It is called by the custom module loading machinery to rewrite @@ -184,6 +184,7 @@ REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION OR CACHED FILES WON'T GET INVALIDATED PROPERLY. """ + futureimpline = False for i, t in enumerate(tokens): # Convert most string literals to byte literals. String literals # in Python 2 are bytes. String literals in Python 3 are unicode. @@ -217,6 +218,29 @@ t.line) continue + # Insert compatibility imports at "from __future__ import" line. + # No '\n' should be added to preserve line numbers. + if (t.type == token.NAME and t.string == 'import' and + all(u.type == token.NAME for u in tokens[i - 2:i]) and + [u.string for u in tokens[i - 2:i]] == ['from', '__future__']): + futureimpline = True + if t.type == token.NEWLINE and futureimpline: + futureimpline = False + if fullname == 'mercurial.pycompat': + yield t + continue + r, c = t.start + l = (b'; from mercurial.pycompat import ' + b'delattr, getattr, hasattr, setattr, xrange\n') + for u in tokenize.tokenize(io.BytesIO(l).readline): + if u.type in (tokenize.ENCODING, token.ENDMARKER): + continue + yield tokenize.TokenInfo(u.type, u.string, + (r, c + u.start[1]), + (r, c + u.end[1]), + '') + continue + try: nexttoken = tokens[i + 1] except IndexError: @@ -279,7 +303,7 @@ # ``replacetoken`` or any mechanism that changes semantics of module # loading is changed. Otherwise cached bytecode may get loaded without # the new transformation mechanisms applied. - BYTECODEHEADER = b'HG\x00\x01' + BYTECODEHEADER = b'HG\x00\x02' class hgloader(importlib.machinery.SourceFileLoader): """Custom module loader that transforms source code. @@ -338,7 +362,7 @@ """Perform token transformation before compilation.""" buf = io.BytesIO(data) tokens = tokenize.tokenize(buf.readline) - data = tokenize.untokenize(replacetokens(list(tokens))) + data = tokenize.untokenize(replacetokens(list(tokens), self.name)) # Python's built-in importer strips frames from exceptions raised # for this code. Unfortunately, that mechanism isn't extensible # and our frame will be blamed for the import failure. There diff -r 45fa8de47a0f -r 178c89e8519a mercurial/pycompat.py --- a/mercurial/pycompat.py Sun Aug 14 12:51:21 2016 +0900 +++ b/mercurial/pycompat.py Tue Aug 16 12:35:15 2016 +0900 @@ -32,7 +32,6 @@ if sys.version_info[0] >= 3: import builtins import functools - builtins.xrange = range def _wrapattrfunc(f): @functools.wraps(f) @@ -42,10 +41,12 @@ return f(object, name, *args) return w + # these wrappers are automagically imported by hgloader delattr = _wrapattrfunc(builtins.delattr) getattr = _wrapattrfunc(builtins.getattr) hasattr = _wrapattrfunc(builtins.hasattr) setattr = _wrapattrfunc(builtins.setattr) + xrange = builtins.range stringio = io.StringIO empty = _queue.Empty diff -r 45fa8de47a0f -r 178c89e8519a tests/test-check-py3-compat.t --- a/tests/test-check-py3-compat.t Sun Aug 14 12:51:21 2016 +0900 +++ b/tests/test-check-py3-compat.t Tue Aug 16 12:35:15 2016 +0900 @@ -48,7 +48,7 @@ hgext/gpg.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) hgext/graphlog.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) hgext/hgk.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) - hgext/highlight/highlight.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) + hgext/highlight/highlight.py: error importing module: No module named 'pygments' (line 13) hgext/histedit.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) hgext/journal.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) hgext/keyword.py: error importing: str expected, not bytes (error at encoding.py:*) (glob) @@ -122,52 +122,51 @@ mercurial/hook.py: error importing: str expected, not bytes (error at i18n.py:*) (glob) mercurial/httpconnection.py: error importing: str expected, not bytes (error at i18n.py:*) (glob) mercurial/httppeer.py: error importing: str expected, not bytes (error at i18n.py:*) (glob) - mercurial/keepalive.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/localrepo.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/lock.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/mail.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/manifest.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/match.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/mdiff.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/merge.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/minirst.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/namespaces.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/obsolete.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/patch.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/pathutil.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/peer.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/pure/mpatch.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/pure/parsers.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/pushkey.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/pvec.py: error importing: getattr(): attribute name must be string (error at pycompat.py:*) (glob) - mercurial/registrar.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/repair.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/repoview.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/revlog.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/revset.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/scmutil.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/scmwindows.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/similar.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/simplemerge.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/sshpeer.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/sshserver.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/sslutil.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/statichttprepo.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/store.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/streamclone.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/subrepo.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/tagmerge.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/tags.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/templatefilters.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/templatekw.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/templater.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/transaction.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/ui.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/unionrepo.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/url.py: error importing: getattr(): attribute name must be string (error at util.py:*) (glob) - mercurial/verify.py: error importing: attribute name must be string, not 'bytes' (error at mdiff.py:*) (glob) + mercurial/keepalive.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/localrepo.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/lock.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/mail.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/manifest.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/match.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/mdiff.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/merge.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/minirst.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/namespaces.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/obsolete.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/patch.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/pathutil.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/peer.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/profiling.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/pushkey.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/pvec.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/registrar.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/repair.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/repoview.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/revlog.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/revset.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/scmutil.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/scmwindows.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/similar.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/simplemerge.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/sshpeer.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/sshserver.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/sslutil.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/statichttprepo.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/store.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/streamclone.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/subrepo.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/tagmerge.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/tags.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/templatefilters.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/templatekw.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/templater.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/transaction.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/ui.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/unionrepo.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/url.py: error importing: __slots__ items must be strings, not 'bytes' (error at util.py:558) + mercurial/verify.py: error importing module: unorderable types: str() >= tuple() (line 884) mercurial/win32.py: error importing module: No module named 'msvcrt' (line *) (glob) mercurial/windows.py: error importing module: No module named 'msvcrt' (line *) (glob) - mercurial/wireproto.py: error importing module: a bytes-like object is required, not 'str' (line *) (glob) + mercurial/wireproto.py: error importing module: unorderable types: str() >= tuple() (line 884) #endif