author | Gregory Szorc <gregory.szorc@gmail.com> |
Sat, 14 Apr 2018 20:04:59 -0700 | |
changeset 37758 | 9b3a348c9b2f |
parent 36617 | 5246f940a48e |
child 38785 | fb9121ea38c4 |
permissions | -rw-r--r-- |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
1 |
# __init__.py - Startup and module loading logic for Mercurial. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
2 |
# |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
3 |
# Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 |
# |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 |
# This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 |
# GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 |
|
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 |
from __future__ import absolute_import |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
|
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
10 |
import sys |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
28513
diff
changeset
|
11 |
|
32420
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
12 |
# Allow 'from mercurial import demandimport' to keep working. |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
13 |
import hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
14 |
demandimport = hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
15 |
|
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
16 |
__all__ = [] |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
17 |
|
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
18 |
# Python 3 uses a custom module loader that transforms source code between |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
19 |
# source file reading and compilation. This is done by registering a custom |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 |
# finder that changes the spec for Mercurial modules to use a custom loader. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
21 |
if sys.version_info[0] >= 3: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
22 |
import importlib |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
23 |
import importlib.abc |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
24 |
import io |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
25 |
import token |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
26 |
import tokenize |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
27 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
28 |
class hgpathentryfinder(importlib.abc.MetaPathFinder): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
29 |
"""A sys.meta_path finder that uses a custom module loader.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
30 |
def find_spec(self, fullname, path, target=None): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
31 |
# Only handle Mercurial-related modules. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
32 |
if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
33 |
return None |
36617
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
34 |
# don't try to parse binary |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
35 |
if fullname.startswith('mercurial.cext.'): |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
36 |
return None |
34396
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
37 |
# third-party packages are expected to be dual-version clean |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
38 |
if fullname.startswith('mercurial.thirdparty'): |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
39 |
return None |
31307
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
40 |
# zstd is already dual-version clean, don't try and mangle it |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
41 |
if fullname.startswith('mercurial.zstd'): |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
42 |
return None |
32521
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
43 |
# pywatchman is already dual-version clean, don't try and mangle it |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
44 |
if fullname.startswith('hgext.fsmonitor.pywatchman'): |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
45 |
return None |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
46 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
47 |
# Try to find the module using other registered finders. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
48 |
spec = None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
49 |
for finder in sys.meta_path: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
50 |
if finder == self: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
51 |
continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
52 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
53 |
spec = finder.find_spec(fullname, path, target=target) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
54 |
if spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
55 |
break |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
56 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
57 |
# This is a Mercurial-related module but we couldn't find it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
58 |
# using the previously-registered finders. This likely means |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
59 |
# the module doesn't exist. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
60 |
if not spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
61 |
return None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
62 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
63 |
# TODO need to support loaders from alternate specs, like zip |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
64 |
# loaders. |
32425
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
65 |
loader = hgloader(spec.name, spec.origin) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
66 |
# Can't use util.safehasattr here because that would require |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
67 |
# importing util, and we're in import code. |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
68 |
if hasattr(spec.loader, 'loader'): # hasattr-py3-only |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
69 |
# This is a nested loader (maybe a lazy loader?) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
70 |
spec.loader.loader = loader |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
71 |
else: |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
72 |
spec.loader = loader |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
73 |
return spec |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
74 |
|
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
75 |
def replacetokens(tokens, fullname): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
76 |
"""Transform a stream of tokens from raw to Python 3. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
77 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
78 |
It is called by the custom module loading machinery to rewrite |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
79 |
source/tokens between source decoding and compilation. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
80 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
81 |
Returns a generator of possibly rewritten tokens. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
82 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
83 |
The input token list may be mutated as part of processing. However, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
84 |
its changes do not necessarily match the output token stream. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
85 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
86 |
REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
87 |
OR CACHED FILES WON'T GET INVALIDATED PROPERLY. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
88 |
""" |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
89 |
futureimpline = False |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
90 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
91 |
# The following utility functions access the tokens list and i index of |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
92 |
# the for i, t enumerate(tokens) loop below |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
93 |
def _isop(j, *o): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
94 |
"""Assert that tokens[j] is an OP with one of the given values""" |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
95 |
try: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
96 |
return tokens[j].type == token.OP and tokens[j].string in o |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
97 |
except IndexError: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
98 |
return False |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
99 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
100 |
def _findargnofcall(n): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
101 |
"""Find arg n of a call expression (start at 0) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
102 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
103 |
Returns index of the first token of that argument, or None if |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
104 |
there is not that many arguments. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
105 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
106 |
Assumes that token[i + 1] is '('. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
107 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
108 |
""" |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
109 |
nested = 0 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
110 |
for j in range(i + 2, len(tokens)): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
111 |
if _isop(j, ')', ']', '}'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
112 |
# end of call, tuple, subscription or dict / set |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
113 |
nested -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
114 |
if nested < 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
115 |
return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
116 |
elif n == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
117 |
# this is the starting position of arg |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
118 |
return j |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
119 |
elif _isop(j, '(', '[', '{'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
120 |
nested += 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
121 |
elif _isop(j, ',') and nested == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
122 |
n -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
123 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
124 |
return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
125 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
126 |
def _ensureunicode(j): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
127 |
"""Make sure the token at j is a unicode string |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
128 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
129 |
This rewrites a string token to include the unicode literal prefix |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
130 |
so the string transformer won't add the byte prefix. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
131 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
132 |
Ignores tokens that are not strings. Assumes bounds checking has |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
133 |
already been done. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
134 |
|
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
135 |
""" |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
136 |
st = tokens[j] |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
137 |
if st.type == token.STRING and st.string.startswith(("'", '"')): |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
138 |
tokens[j] = st._replace(string='u%s' % st.string) |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
139 |
|
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
140 |
for i, t in enumerate(tokens): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
141 |
# Convert most string literals to byte literals. String literals |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
142 |
# in Python 2 are bytes. String literals in Python 3 are unicode. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
143 |
# Most strings in Mercurial are bytes and unicode strings are rare. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
144 |
# Rather than rewrite all string literals to use ``b''`` to indicate |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
145 |
# byte strings, we apply this token transformer to insert the ``b`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
146 |
# prefix nearly everywhere. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
147 |
if t.type == token.STRING: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
148 |
s = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
149 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
150 |
# Preserve docstrings as string literals. This is inconsistent |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
151 |
# with regular unprefixed strings. However, the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
152 |
# "from __future__" parsing (which allows a module docstring to |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
153 |
# exist before it) doesn't properly handle the docstring if it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
154 |
# is b''' prefixed, leading to a SyntaxError. We leave all |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
155 |
# docstrings as unprefixed to avoid this. This means Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
156 |
# components touching docstrings need to handle unicode, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
157 |
# unfortunately. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
158 |
if s[0:3] in ("'''", '"""'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
159 |
yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
160 |
continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
161 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
162 |
# If the first character isn't a quote, it is likely a string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
163 |
# prefixing character (such as 'b', 'u', or 'r'. Ignore. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
164 |
if s[0] not in ("'", '"'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
165 |
yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
166 |
continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
167 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
168 |
# String literal. Prefix to make a b'' string. |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
169 |
yield t._replace(string='b%s' % t.string) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
170 |
continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
171 |
|
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
172 |
# Insert compatibility imports at "from __future__ import" line. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
173 |
# No '\n' should be added to preserve line numbers. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
174 |
if (t.type == token.NAME and t.string == 'import' and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
175 |
all(u.type == token.NAME for u in tokens[i - 2:i]) and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
176 |
[u.string for u in tokens[i - 2:i]] == ['from', '__future__']): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
177 |
futureimpline = True |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
178 |
if t.type == token.NEWLINE and futureimpline: |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
179 |
futureimpline = False |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
180 |
if fullname == 'mercurial.pycompat': |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
181 |
yield t |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
182 |
continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
183 |
r, c = t.start |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
184 |
l = (b'; from mercurial.pycompat import ' |
31843
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
185 |
b'delattr, getattr, hasattr, setattr, xrange, ' |
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
186 |
b'open, unicode\n') |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
187 |
for u in tokenize.tokenize(io.BytesIO(l).readline): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
188 |
if u.type in (tokenize.ENCODING, token.ENDMARKER): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
189 |
continue |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
190 |
yield u._replace( |
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
191 |
start=(r, c + u.start[1]), end=(r, c + u.end[1])) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
192 |
continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
193 |
|
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
194 |
# This looks like a function call. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
195 |
if t.type == token.NAME and _isop(i + 1, '('): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
196 |
fn = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
197 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
198 |
# *attr() builtins don't accept byte strings to 2nd argument. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
199 |
if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
200 |
not _isop(i - 1, '.')): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
201 |
arg1idx = _findargnofcall(1) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
202 |
if arg1idx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
203 |
_ensureunicode(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
205 |
# .encode() and .decode() on str/bytes/unicode don't accept |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
206 |
# byte strings on Python 3. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
207 |
elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
208 |
for argn in range(2): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
209 |
argidx = _findargnofcall(argn) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
210 |
if argidx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
211 |
_ensureunicode(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
212 |
|
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
213 |
# It changes iteritems/values to items/values as they are not |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
214 |
# present in Python 3 world. |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
215 |
elif fn in ('iteritems', 'itervalues'): |
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
216 |
yield t._replace(string=fn[4:]) |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
217 |
continue |
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
218 |
|
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
219 |
# Emit unmodified token. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
220 |
yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
221 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
222 |
# Header to add to bytecode files. This MUST be changed when |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
223 |
# ``replacetoken`` or any mechanism that changes semantics of module |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
224 |
# loading is changed. Otherwise cached bytecode may get loaded without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
225 |
# the new transformation mechanisms applied. |
31843
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
226 |
BYTECODEHEADER = b'HG\x00\x0a' |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
227 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
228 |
class hgloader(importlib.machinery.SourceFileLoader): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
229 |
"""Custom module loader that transforms source code. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
230 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
231 |
When the source code is converted to a code object, we transform |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
232 |
certain patterns to be Python 3 compatible. This allows us to write code |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
233 |
that is natively Python 2 and compatible with Python 3 without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
234 |
making the code excessively ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
235 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
236 |
We do this by transforming the token stream between parse and compile. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
237 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
238 |
Implementing transformations invalidates caching assumptions made |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
239 |
by the built-in importer. The built-in importer stores a header on |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
240 |
saved bytecode files indicating the Python/bytecode version. If the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
241 |
version changes, the cached bytecode is ignored. The Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
242 |
transformations could change at any time. This means we need to check |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
243 |
that cached bytecode was generated with the current transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
244 |
code or there could be a mismatch between cached bytecode and what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
245 |
would be generated from this class. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
246 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
247 |
We supplement the bytecode caching layer by wrapping ``get_data`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
248 |
and ``set_data``. These functions are called when the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
249 |
``SourceFileLoader`` retrieves and saves bytecode cache files, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
250 |
respectively. We simply add an additional header on the file. As |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
251 |
long as the version in this file is changed when semantics change, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
252 |
cached bytecode should be invalidated when transformations change. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
253 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
254 |
The added header has the form ``HG<VERSION>``. That is a literal |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
255 |
``HG`` with 2 binary bytes indicating the transformation version. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
256 |
""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
257 |
def get_data(self, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
258 |
data = super(hgloader, self).get_data(path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
259 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
260 |
if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
261 |
return data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
262 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
263 |
# There should be a header indicating the Mercurial transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
264 |
# version. If it doesn't exist or doesn't match the current version, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
265 |
# we raise an OSError because that is what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
266 |
# ``SourceFileLoader.get_code()`` expects when loading bytecode |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
267 |
# paths to indicate the cached file is "bad." |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
268 |
if data[0:2] != b'HG': |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
269 |
raise OSError('no hg header') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
270 |
if data[0:4] != BYTECODEHEADER: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
271 |
raise OSError('hg header version mismatch') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
272 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
273 |
return data[4:] |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
274 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
275 |
def set_data(self, path, data, *args, **kwargs): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
276 |
if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
277 |
data = BYTECODEHEADER + data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
278 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
279 |
return super(hgloader, self).set_data(path, data, *args, **kwargs) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
280 |
|
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
281 |
def source_to_code(self, data, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
282 |
"""Perform token transformation before compilation.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
283 |
buf = io.BytesIO(data) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
284 |
tokens = tokenize.tokenize(buf.readline) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
285 |
data = tokenize.untokenize(replacetokens(list(tokens), self.name)) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
286 |
# Python's built-in importer strips frames from exceptions raised |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
287 |
# for this code. Unfortunately, that mechanism isn't extensible |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
288 |
# and our frame will be blamed for the import failure. There |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
289 |
# are extremely hacky ways to do frame stripping. We haven't |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
290 |
# implemented them because they are very ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
291 |
return super(hgloader, self).source_to_code(data, path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
292 |
|
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
293 |
# We automagically register our custom importer as a side-effect of |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
294 |
# loading. This is necessary to ensure that any entry points are able |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
295 |
# to import mercurial.* modules without having to perform this |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
296 |
# registration themselves. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
297 |
if not any(isinstance(x, hgpathentryfinder) for x in sys.meta_path): |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
298 |
# meta_path is used before any implicit finders and before sys.path. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
299 |
sys.meta_path.insert(0, hgpathentryfinder()) |