comparison mercurial/__init__.py @ 29800:178c89e8519a

py3: import builtin wrappers automagically by code transformer

This should be less invasive than mucking builtins.

Since tokenize.untokenize() looks start/end positions of tokens, we calculates
them from the NEWLINE token of the future import.

29799:45fa8de47a0f

            # TODO need to support loaders from alternate specs, like zip
            # loaders.
            spec.loader = hgloader(spec.name, spec.origin)
            return spec

    def replacetokens(tokens):
        """Transform a stream of tokens from raw to Python 3.

        It is called by the custom module loading machinery to rewrite
        source/tokens between source decoding and compilation.

        its changes do not necessarily match the output token stream.

        REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
        OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
        """
        for i, t in enumerate(tokens):
            # Convert most string literals to byte literals. String literals
            # in Python 2 are bytes. String literals in Python 3 are unicode.
            # Most strings in Mercurial are bytes and unicode strings are rare.
            # Rather than rewrite all string literals to use ``b''`` to indicate

                    continue

                # String literal. Prefix to make a b'' string.
                yield tokenize.TokenInfo(t.type, 'b%s' % s, t.start, t.end,
                                          t.line)
                continue

            try:
                nexttoken = tokens[i + 1]
            except IndexError:

    # Header to add to bytecode files. This MUST be changed when
    # ``replacetoken`` or any mechanism that changes semantics of module
    # loading is changed. Otherwise cached bytecode may get loaded without
    # the new transformation mechanisms applied.
    BYTECODEHEADER = b'HG\x00\x01'

    class hgloader(importlib.machinery.SourceFileLoader):
        """Custom module loader that transforms source code.

        When the source code is converted to a code object, we transform

        def source_to_code(self, data, path):
            """Perform token transformation before compilation."""
            buf = io.BytesIO(data)
            tokens = tokenize.tokenize(buf.readline)
            data = tokenize.untokenize(replacetokens(list(tokens)))
            # Python's built-in importer strips frames from exceptions raised
            # for this code. Unfortunately, that mechanism isn't extensible
            # and our frame will be blamed for the import failure. There
            # are extremely hacky ways to do frame stripping. We haven't
            # implemented them because they are very ugly.

29800:178c89e8519a

            # TODO need to support loaders from alternate specs, like zip
            # loaders.
            spec.loader = hgloader(spec.name, spec.origin)
            return spec

    def replacetokens(tokens, fullname):
        """Transform a stream of tokens from raw to Python 3.

        It is called by the custom module loading machinery to rewrite
        source/tokens between source decoding and compilation.

        its changes do not necessarily match the output token stream.

        REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
        OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
        """
        futureimpline = False
        for i, t in enumerate(tokens):
            # Convert most string literals to byte literals. String literals
            # in Python 2 are bytes. String literals in Python 3 are unicode.
            # Most strings in Mercurial are bytes and unicode strings are rare.
            # Rather than rewrite all string literals to use ``b''`` to indicate

                    continue

                # String literal. Prefix to make a b'' string.
                yield tokenize.TokenInfo(t.type, 'b%s' % s, t.start, t.end,
                                          t.line)
                continue

            # Insert compatibility imports at "from __future__ import" line.
            # No '\n' should be added to preserve line numbers.
            if (t.type == token.NAME and t.string == 'import' and
                all(u.type == token.NAME for u in tokens[i - 2:i]) and
                [u.string for u in tokens[i - 2:i]] == ['from', '__future__']):
                futureimpline = True
            if t.type == token.NEWLINE and futureimpline:
                futureimpline = False
                if fullname == 'mercurial.pycompat':
                    yield t
                    continue
                r, c = t.start
                l = (b'; from mercurial.pycompat import '
                     b'delattr, getattr, hasattr, setattr, xrange\n')
                for u in tokenize.tokenize(io.BytesIO(l).readline):
                    if u.type in (tokenize.ENCODING, token.ENDMARKER):
                        continue
                    yield tokenize.TokenInfo(u.type, u.string,
                                             (r, c + u.start[1]),
                                             (r, c + u.end[1]),
                                             '')
                continue

            try:
                nexttoken = tokens[i + 1]
            except IndexError:

    # Header to add to bytecode files. This MUST be changed when
    # ``replacetoken`` or any mechanism that changes semantics of module
    # loading is changed. Otherwise cached bytecode may get loaded without
    # the new transformation mechanisms applied.
    BYTECODEHEADER = b'HG\x00\x02'

    class hgloader(importlib.machinery.SourceFileLoader):
        """Custom module loader that transforms source code.

        When the source code is converted to a code object, we transform

        def source_to_code(self, data, path):
            """Perform token transformation before compilation."""
            buf = io.BytesIO(data)
            tokens = tokenize.tokenize(buf.readline)
            data = tokenize.untokenize(replacetokens(list(tokens), self.name))
            # Python's built-in importer strips frames from exceptions raised
            # for this code. Unfortunately, that mechanism isn't extensible
            # and our frame will be blamed for the import failure. There
            # are extremely hacky ways to do frame stripping. We haven't
            # implemented them because they are very ugly.