mercurial/__init__.py
author Pierre-Yves David <pierre-yves.david@ens-lyon.org>
Thu, 03 Nov 2016 05:10:14 +0100
changeset 30280 0269ab4f4371
parent 30166 102e6ef5bb3a
child 31149 76a64c1e5439
permissions -rw-r--r--
tests: merge 'test-push-validation.t' into 'test-push.t' That test file is very small and is merge with the new 'test-push.t'. No logic is changed but repository name are update to avoid collision. We don't register this as a copy because is actually a "ypoc" merging two file together without replacing the destination and Mercurial cannot express that.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     1
# __init__.py - Startup and module loading logic for Mercurial.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     2
#
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     3
# Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com>
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     4
#
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     5
# This software may be used and distributed according to the terms of the
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     6
# GNU General Public License version 2 or any later version.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     7
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     8
from __future__ import absolute_import
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     9
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    10
import imp
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    11
import os
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    12
import sys
27225
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    13
import zipimport
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    14
29266
b3a677c82a35 debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents: 28513
diff changeset
    15
from . import (
b3a677c82a35 debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents: 28513
diff changeset
    16
    policy
b3a677c82a35 debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents: 28513
diff changeset
    17
)
b3a677c82a35 debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents: 28513
diff changeset
    18
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    19
__all__ = []
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    20
29266
b3a677c82a35 debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents: 28513
diff changeset
    21
modulepolicy = policy.policy
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    22
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    23
# Modules that have both Python and C implementations. See also the
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    24
# set of .py files under mercurial/pure/.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    25
_dualmodules = set([
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    26
    'mercurial.base85',
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    27
    'mercurial.bdiff',
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    28
    'mercurial.diffhelpers',
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    29
    'mercurial.mpatch',
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    30
    'mercurial.osutil',
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    31
    'mercurial.parsers',
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    32
])
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    33
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    34
class hgimporter(object):
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    35
    """Object that conforms to import hook interface defined in PEP-302."""
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    36
    def find_module(self, name, path=None):
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    37
        # We only care about modules that have both C and pure implementations.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    38
        if name in _dualmodules:
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    39
            return self
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    40
        return None
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    41
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    42
    def load_module(self, name):
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    43
        mod = sys.modules.get(name, None)
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    44
        if mod:
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    45
            return mod
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    46
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    47
        mercurial = sys.modules['mercurial']
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    48
27225
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    49
        # The zip importer behaves sufficiently differently from the default
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    50
        # importer to warrant its own code path.
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    51
        loader = getattr(mercurial, '__loader__', None)
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    52
        if isinstance(loader, zipimport.zipimporter):
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    53
            def ziploader(*paths):
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    54
                """Obtain a zipimporter for a directory under the main zip."""
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    55
                path = os.path.join(loader.archive, *paths)
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    56
                zl = sys.path_importer_cache.get(path)
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    57
                if not zl:
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    58
                    zl = zipimport.zipimporter(path)
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    59
                return zl
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    60
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    61
            try:
29490
b4d117cee636 policy: add cffi policy for PyPy
Maciej Fijalkowski <fijall@gmail.com>
parents: 29266
diff changeset
    62
                if modulepolicy in policy.policynoc:
27225
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    63
                    raise ImportError()
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    64
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    65
                zl = ziploader('mercurial')
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    66
                mod = zl.load_module(name)
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    67
                # Unlike imp, ziploader doesn't expose module metadata that
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    68
                # indicates the type of module. So just assume what we found
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    69
                # is OK (even though it could be a pure Python module).
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    70
            except ImportError:
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    71
                if modulepolicy == 'c':
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    72
                    raise
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    73
                zl = ziploader('mercurial', 'pure')
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    74
                mod = zl.load_module(name)
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    75
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    76
            sys.modules[name] = mod
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    77
            return mod
30a20167ae29 mercurial: support loading modules from zipimporter
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27224
diff changeset
    78
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    79
        # Unlike the default importer which searches special locations and
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    80
        # sys.path, we only look in the directory where "mercurial" was
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    81
        # imported from.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    82
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    83
        # imp.find_module doesn't support submodules (modules with ".").
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    84
        # Instead you have to pass the parent package's __path__ attribute
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    85
        # as the path argument.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    86
        stem = name.split('.')[-1]
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    87
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    88
        try:
29490
b4d117cee636 policy: add cffi policy for PyPy
Maciej Fijalkowski <fijall@gmail.com>
parents: 29266
diff changeset
    89
            if modulepolicy in policy.policynoc:
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    90
                raise ImportError()
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    91
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    92
            modinfo = imp.find_module(stem, mercurial.__path__)
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    93
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    94
            # The Mercurial installer used to copy files from
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    95
            # mercurial/pure/*.py to mercurial/*.py. Therefore, it's possible
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    96
            # for some installations to have .py files under mercurial/*.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    97
            # Loading Python modules when we expected C versions could result
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    98
            # in a) poor performance b) loading a version from a previous
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    99
            # Mercurial version, potentially leading to incompatibility. Either
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   100
            # scenario is bad. So we verify that modules loaded from
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   101
            # mercurial/* are C extensions. If the current policy allows the
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   102
            # loading of .py modules, the module will be re-imported from
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   103
            # mercurial/pure/* below.
27223
a40c84defd76 mercurial: be more strict about loading dual implemented modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27220
diff changeset
   104
            if modinfo[2][2] != imp.C_EXTENSION:
a40c84defd76 mercurial: be more strict about loading dual implemented modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27220
diff changeset
   105
                raise ImportError('.py version of %s found where C '
a40c84defd76 mercurial: be more strict about loading dual implemented modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27220
diff changeset
   106
                                  'version should exist' % name)
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   107
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   108
        except ImportError:
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   109
            if modulepolicy == 'c':
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   110
                raise
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   111
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   112
            # Could not load the C extension and pure Python is allowed. So
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   113
            # try to load them.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   114
            from . import pure
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   115
            modinfo = imp.find_module(stem, pure.__path__)
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   116
            if not modinfo:
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   117
                raise ImportError('could not find mercurial module %s' %
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   118
                                  name)
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   119
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   120
        mod = imp.load_module(name, *modinfo)
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   121
        sys.modules[name] = mod
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   122
        return mod
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   123
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   124
# Python 3 uses a custom module loader that transforms source code between
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   125
# source file reading and compilation. This is done by registering a custom
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   126
# finder that changes the spec for Mercurial modules to use a custom loader.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   127
if sys.version_info[0] >= 3:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   128
    from . import pure
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   129
    import importlib
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   130
    import io
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   131
    import token
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   132
    import tokenize
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   133
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   134
    class hgpathentryfinder(importlib.abc.MetaPathFinder):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   135
        """A sys.meta_path finder that uses a custom module loader."""
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   136
        def find_spec(self, fullname, path, target=None):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   137
            # Only handle Mercurial-related modules.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   138
            if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   139
                return None
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   140
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   141
            # This assumes Python 3 doesn't support loading C modules.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   142
            if fullname in _dualmodules:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   143
                stem = fullname.split('.')[-1]
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   144
                fullname = 'mercurial.pure.%s' % stem
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   145
                target = pure
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   146
                assert len(path) == 1
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   147
                path = [os.path.join(path[0], 'pure')]
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   148
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   149
            # Try to find the module using other registered finders.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   150
            spec = None
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   151
            for finder in sys.meta_path:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   152
                if finder == self:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   153
                    continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   154
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   155
                spec = finder.find_spec(fullname, path, target=target)
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   156
                if spec:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   157
                    break
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   158
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   159
            # This is a Mercurial-related module but we couldn't find it
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   160
            # using the previously-registered finders. This likely means
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   161
            # the module doesn't exist.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   162
            if not spec:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   163
                return None
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   164
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   165
            if fullname.startswith('mercurial.pure.'):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   166
                spec.name = spec.name.replace('.pure.', '.')
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   167
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   168
            # TODO need to support loaders from alternate specs, like zip
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   169
            # loaders.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   170
            spec.loader = hgloader(spec.name, spec.origin)
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   171
            return spec
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   172
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   173
    def replacetokens(tokens, fullname):
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   174
        """Transform a stream of tokens from raw to Python 3.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   175
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   176
        It is called by the custom module loading machinery to rewrite
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   177
        source/tokens between source decoding and compilation.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   178
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   179
        Returns a generator of possibly rewritten tokens.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   180
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   181
        The input token list may be mutated as part of processing. However,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   182
        its changes do not necessarily match the output token stream.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   183
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   184
        REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   185
        OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   186
        """
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   187
        futureimpline = False
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   188
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   189
        # The following utility functions access the tokens list and i index of
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   190
        # the for i, t enumerate(tokens) loop below
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   191
        def _isop(j, *o):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   192
            """Assert that tokens[j] is an OP with one of the given values"""
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   193
            try:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   194
                return tokens[j].type == token.OP and tokens[j].string in o
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   195
            except IndexError:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   196
                return False
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   197
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   198
        def _findargnofcall(n):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   199
            """Find arg n of a call expression (start at 0)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   200
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   201
            Returns index of the first token of that argument, or None if
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   202
            there is not that many arguments.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   203
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   204
            Assumes that token[i + 1] is '('.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   205
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   206
            """
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   207
            nested = 0
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   208
            for j in range(i + 2, len(tokens)):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   209
                if _isop(j, ')', ']', '}'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   210
                    # end of call, tuple, subscription or dict / set
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   211
                    nested -= 1
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   212
                    if nested < 0:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   213
                        return None
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   214
                elif n == 0:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   215
                    # this is the starting position of arg
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   216
                    return j
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   217
                elif _isop(j, '(', '[', '{'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   218
                    nested += 1
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   219
                elif _isop(j, ',') and nested == 0:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   220
                    n -= 1
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   221
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   222
            return None
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   223
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   224
        def _ensureunicode(j):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   225
            """Make sure the token at j is a unicode string
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   226
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   227
            This rewrites a string token to include the unicode literal prefix
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   228
            so the string transformer won't add the byte prefix.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   229
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   230
            Ignores tokens that are not strings. Assumes bounds checking has
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   231
            already been done.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   232
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   233
            """
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   234
            st = tokens[j]
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   235
            if st.type == token.STRING and st.string.startswith(("'", '"')):
30166
102e6ef5bb3a py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents: 30165
diff changeset
   236
                tokens[j] = st._replace(string='u%s' % st.string)
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   237
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   238
        for i, t in enumerate(tokens):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   239
            # Convert most string literals to byte literals. String literals
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   240
            # in Python 2 are bytes. String literals in Python 3 are unicode.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   241
            # Most strings in Mercurial are bytes and unicode strings are rare.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   242
            # Rather than rewrite all string literals to use ``b''`` to indicate
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   243
            # byte strings, we apply this token transformer to insert the ``b``
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   244
            # prefix nearly everywhere.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   245
            if t.type == token.STRING:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   246
                s = t.string
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   247
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   248
                # Preserve docstrings as string literals. This is inconsistent
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   249
                # with regular unprefixed strings. However, the
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   250
                # "from __future__" parsing (which allows a module docstring to
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   251
                # exist before it) doesn't properly handle the docstring if it
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   252
                # is b''' prefixed, leading to a SyntaxError. We leave all
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   253
                # docstrings as unprefixed to avoid this. This means Mercurial
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   254
                # components touching docstrings need to handle unicode,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   255
                # unfortunately.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   256
                if s[0:3] in ("'''", '"""'):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   257
                    yield t
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   258
                    continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   259
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   260
                # If the first character isn't a quote, it is likely a string
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   261
                # prefixing character (such as 'b', 'u', or 'r'. Ignore.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   262
                if s[0] not in ("'", '"'):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   263
                    yield t
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   264
                    continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   265
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   266
                # String literal. Prefix to make a b'' string.
30166
102e6ef5bb3a py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents: 30165
diff changeset
   267
                yield t._replace(string='b%s' % t.string)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   268
                continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   269
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   270
            # Insert compatibility imports at "from __future__ import" line.
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   271
            # No '\n' should be added to preserve line numbers.
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   272
            if (t.type == token.NAME and t.string == 'import' and
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   273
                all(u.type == token.NAME for u in tokens[i - 2:i]) and
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   274
                [u.string for u in tokens[i - 2:i]] == ['from', '__future__']):
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   275
                futureimpline = True
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   276
            if t.type == token.NEWLINE and futureimpline:
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   277
                futureimpline = False
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   278
                if fullname == 'mercurial.pycompat':
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   279
                    yield t
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   280
                    continue
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   281
                r, c = t.start
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   282
                l = (b'; from mercurial.pycompat import '
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   283
                     b'delattr, getattr, hasattr, setattr, xrange\n')
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   284
                for u in tokenize.tokenize(io.BytesIO(l).readline):
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   285
                    if u.type in (tokenize.ENCODING, token.ENDMARKER):
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   286
                        continue
30166
102e6ef5bb3a py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents: 30165
diff changeset
   287
                    yield u._replace(
102e6ef5bb3a py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents: 30165
diff changeset
   288
                        start=(r, c + u.start[1]), end=(r, c + u.end[1]))
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   289
                continue
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   290
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   291
            # This looks like a function call.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   292
            if t.type == token.NAME and _isop(i + 1, '('):
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   293
                fn = t.string
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   294
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   295
                # *attr() builtins don't accept byte strings to 2nd argument.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   296
                if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   297
                        not _isop(i - 1, '.')):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   298
                    arg1idx = _findargnofcall(1)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   299
                    if arg1idx is not None:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   300
                        _ensureunicode(arg1idx)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   301
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   302
                # .encode() and .decode() on str/bytes/unicode don't accept
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   303
                # byte strings on Python 3.
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   304
                elif fn in ('encode', 'decode') and _isop(i - 1, '.'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   305
                    for argn in range(2):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   306
                        argidx = _findargnofcall(argn)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   307
                        if argidx is not None:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   308
                            _ensureunicode(argidx)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   309
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   310
                # Bare open call (not an attribute on something else), the
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   311
                # second argument (mode) must be a string, not bytes
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   312
                elif fn == 'open' and not _isop(i - 1, '.'):
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   313
                    arg1idx = _findargnofcall(1)
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   314
                    if arg1idx is not None:
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   315
                        _ensureunicode(arg1idx)
30118
c19266edd93e py3: a second argument to open can't be bytes
Martijn Pieters <mjpieters@fb.com>
parents: 30052
diff changeset
   316
30052
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
   317
                # It changes iteritems to items as iteritems is not
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
   318
                # present in Python 3 world.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   319
                elif fn == 'iteritems':
30166
102e6ef5bb3a py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents: 30165
diff changeset
   320
                    yield t._replace(string='items')
30052
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
   321
                    continue
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
   322
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   323
            # Emit unmodified token.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   324
            yield t
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   325
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   326
    # Header to add to bytecode files. This MUST be changed when
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   327
    # ``replacetoken`` or any mechanism that changes semantics of module
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   328
    # loading is changed. Otherwise cached bytecode may get loaded without
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   329
    # the new transformation mechanisms applied.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   330
    BYTECODEHEADER = b'HG\x00\x06'
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   331
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   332
    class hgloader(importlib.machinery.SourceFileLoader):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   333
        """Custom module loader that transforms source code.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   334
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   335
        When the source code is converted to a code object, we transform
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   336
        certain patterns to be Python 3 compatible. This allows us to write code
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   337
        that is natively Python 2 and compatible with Python 3 without
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   338
        making the code excessively ugly.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   339
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   340
        We do this by transforming the token stream between parse and compile.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   341
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   342
        Implementing transformations invalidates caching assumptions made
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   343
        by the built-in importer. The built-in importer stores a header on
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   344
        saved bytecode files indicating the Python/bytecode version. If the
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   345
        version changes, the cached bytecode is ignored. The Mercurial
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   346
        transformations could change at any time. This means we need to check
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   347
        that cached bytecode was generated with the current transformation
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   348
        code or there could be a mismatch between cached bytecode and what
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   349
        would be generated from this class.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   350
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   351
        We supplement the bytecode caching layer by wrapping ``get_data``
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   352
        and ``set_data``. These functions are called when the
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   353
        ``SourceFileLoader`` retrieves and saves bytecode cache files,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   354
        respectively. We simply add an additional header on the file. As
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   355
        long as the version in this file is changed when semantics change,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   356
        cached bytecode should be invalidated when transformations change.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   357
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   358
        The added header has the form ``HG<VERSION>``. That is a literal
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   359
        ``HG`` with 2 binary bytes indicating the transformation version.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   360
        """
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   361
        def get_data(self, path):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   362
            data = super(hgloader, self).get_data(path)
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   363
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   364
            if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   365
                return data
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   366
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   367
            # There should be a header indicating the Mercurial transformation
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   368
            # version. If it doesn't exist or doesn't match the current version,
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   369
            # we raise an OSError because that is what
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   370
            # ``SourceFileLoader.get_code()`` expects when loading bytecode
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   371
            # paths to indicate the cached file is "bad."
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   372
            if data[0:2] != b'HG':
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   373
                raise OSError('no hg header')
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   374
            if data[0:4] != BYTECODEHEADER:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   375
                raise OSError('hg header version mismatch')
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   376
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   377
            return data[4:]
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   378
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   379
        def set_data(self, path, data, *args, **kwargs):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   380
            if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   381
                data = BYTECODEHEADER + data
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   382
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   383
            return super(hgloader, self).set_data(path, data, *args, **kwargs)
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   384
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   385
        def source_to_code(self, data, path):
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   386
            """Perform token transformation before compilation."""
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   387
            buf = io.BytesIO(data)
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   388
            tokens = tokenize.tokenize(buf.readline)
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29550
diff changeset
   389
            data = tokenize.untokenize(replacetokens(list(tokens), self.name))
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   390
            # Python's built-in importer strips frames from exceptions raised
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   391
            # for this code. Unfortunately, that mechanism isn't extensible
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   392
            # and our frame will be blamed for the import failure. There
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   393
            # are extremely hacky ways to do frame stripping. We haven't
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   394
            # implemented them because they are very ugly.
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   395
            return super(hgloader, self).source_to_code(data, path)
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   396
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   397
# We automagically register our custom importer as a side-effect of loading.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   398
# This is necessary to ensure that any entry points are able to import
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   399
# mercurial.* modules without having to perform this registration themselves.
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   400
if sys.version_info[0] >= 3:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   401
    _importercls = hgpathentryfinder
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   402
else:
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   403
    _importercls = hgimporter
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   404
if not any(isinstance(x, _importercls) for x in sys.meta_path):
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
   405
    # meta_path is used before any implicit finders and before sys.path.
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   406
    sys.meta_path.insert(0, _importercls())