Mercurial > hg
annotate mercurial/__init__.py @ 42207:7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Fri, 19 Apr 2019 14:25:18 +0000 |
parents | 4277e20cfec4 |
children | d28d91f9f35a |
rev | line source |
---|---|
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
1 # __init__.py - Startup and module loading logic for Mercurial. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
2 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 from __future__ import absolute_import |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
10 import sys |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
28513
diff
changeset
|
11 |
32420
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
12 # Allow 'from mercurial import demandimport' to keep working. |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
13 import hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
14 demandimport = hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
15 |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
16 __all__ = [] |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
17 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
18 # Python 3 uses a custom module loader that transforms source code between |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
19 # source file reading and compilation. This is done by registering a custom |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 # finder that changes the spec for Mercurial modules to use a custom loader. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
21 if sys.version_info[0] >= 3: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
22 import importlib |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
23 import importlib.abc |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
24 import io |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
25 import token |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
26 import tokenize |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
27 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
28 class hgpathentryfinder(importlib.abc.MetaPathFinder): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
29 """A sys.meta_path finder that uses a custom module loader.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
30 def find_spec(self, fullname, path, target=None): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
31 # Only handle Mercurial-related modules. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
32 if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
33 return None |
36617
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
34 # don't try to parse binary |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
35 if fullname.startswith('mercurial.cext.'): |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
36 return None |
34396
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
37 # third-party packages are expected to be dual-version clean |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
38 if fullname.startswith('mercurial.thirdparty'): |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
39 return None |
31307
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
40 # zstd is already dual-version clean, don't try and mangle it |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
41 if fullname.startswith('mercurial.zstd'): |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
42 return None |
40985
4277e20cfec4
rust-cpython: build and support for Python3
Georges Racinet <gracinet@anybox.fr>
parents:
38785
diff
changeset
|
43 # rustext is built for the right python version, |
4277e20cfec4
rust-cpython: build and support for Python3
Georges Racinet <gracinet@anybox.fr>
parents:
38785
diff
changeset
|
44 # don't try and mangle it |
4277e20cfec4
rust-cpython: build and support for Python3
Georges Racinet <gracinet@anybox.fr>
parents:
38785
diff
changeset
|
45 if fullname.startswith('mercurial.rustext'): |
4277e20cfec4
rust-cpython: build and support for Python3
Georges Racinet <gracinet@anybox.fr>
parents:
38785
diff
changeset
|
46 return None |
32521
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
47 # pywatchman is already dual-version clean, don't try and mangle it |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
48 if fullname.startswith('hgext.fsmonitor.pywatchman'): |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
49 return None |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
50 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
51 # Try to find the module using other registered finders. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
52 spec = None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
53 for finder in sys.meta_path: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
54 if finder == self: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
55 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
56 |
42207
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
57 # Originally the API was a `find_module` method, but it was |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
58 # renamed to `find_spec` in python 3.4, with a new `target` |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
59 # argument. |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
60 find_spec_method = getattr(finder, 'find_spec', None) |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
61 if find_spec_method: |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
62 spec = find_spec_method(fullname, path, target=target) |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
63 else: |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
64 spec = finder.find_module(fullname) |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
65 if spec is not None: |
7ed472e79096
py3: handle meta-path finders that only use pre-python3.4 API
Ludovic Chabant <ludovic@chabant.com>
parents:
40985
diff
changeset
|
66 spec = importlib.util.spec_from_loader(fullname, spec) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
67 if spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
68 break |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
69 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
70 # This is a Mercurial-related module but we couldn't find it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
71 # using the previously-registered finders. This likely means |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
72 # the module doesn't exist. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
73 if not spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
74 return None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
75 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
76 # TODO need to support loaders from alternate specs, like zip |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
77 # loaders. |
32425
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
78 loader = hgloader(spec.name, spec.origin) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
79 # Can't use util.safehasattr here because that would require |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
80 # importing util, and we're in import code. |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
81 if hasattr(spec.loader, 'loader'): # hasattr-py3-only |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
82 # This is a nested loader (maybe a lazy loader?) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
83 spec.loader.loader = loader |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
84 else: |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
85 spec.loader = loader |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
86 return spec |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
87 |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
88 def replacetokens(tokens, fullname): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
89 """Transform a stream of tokens from raw to Python 3. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
90 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
91 It is called by the custom module loading machinery to rewrite |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
92 source/tokens between source decoding and compilation. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
93 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
94 Returns a generator of possibly rewritten tokens. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
95 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
96 The input token list may be mutated as part of processing. However, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
97 its changes do not necessarily match the output token stream. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
98 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
99 REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
100 OR CACHED FILES WON'T GET INVALIDATED PROPERLY. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
101 """ |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
102 futureimpline = False |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
103 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
104 # The following utility functions access the tokens list and i index of |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
105 # the for i, t enumerate(tokens) loop below |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
106 def _isop(j, *o): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
107 """Assert that tokens[j] is an OP with one of the given values""" |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
108 try: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
109 return tokens[j].type == token.OP and tokens[j].string in o |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
110 except IndexError: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
111 return False |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
112 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
113 def _findargnofcall(n): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
114 """Find arg n of a call expression (start at 0) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
115 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
116 Returns index of the first token of that argument, or None if |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
117 there is not that many arguments. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
118 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
119 Assumes that token[i + 1] is '('. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
120 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
121 """ |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
122 nested = 0 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
123 for j in range(i + 2, len(tokens)): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
124 if _isop(j, ')', ']', '}'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
125 # end of call, tuple, subscription or dict / set |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
126 nested -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
127 if nested < 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
128 return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
129 elif n == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
130 # this is the starting position of arg |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
131 return j |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
132 elif _isop(j, '(', '[', '{'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
133 nested += 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
134 elif _isop(j, ',') and nested == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
135 n -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
136 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
137 return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
138 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
139 def _ensureunicode(j): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
140 """Make sure the token at j is a unicode string |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
141 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
142 This rewrites a string token to include the unicode literal prefix |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
143 so the string transformer won't add the byte prefix. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
144 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
145 Ignores tokens that are not strings. Assumes bounds checking has |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
146 already been done. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
147 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
148 """ |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
149 st = tokens[j] |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
150 if st.type == token.STRING and st.string.startswith(("'", '"')): |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
151 tokens[j] = st._replace(string='u%s' % st.string) |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
152 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
153 for i, t in enumerate(tokens): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
154 # Convert most string literals to byte literals. String literals |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
155 # in Python 2 are bytes. String literals in Python 3 are unicode. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
156 # Most strings in Mercurial are bytes and unicode strings are rare. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
157 # Rather than rewrite all string literals to use ``b''`` to indicate |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
158 # byte strings, we apply this token transformer to insert the ``b`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
159 # prefix nearly everywhere. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
160 if t.type == token.STRING: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
161 s = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
162 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
163 # Preserve docstrings as string literals. This is inconsistent |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
164 # with regular unprefixed strings. However, the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
165 # "from __future__" parsing (which allows a module docstring to |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
166 # exist before it) doesn't properly handle the docstring if it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
167 # is b''' prefixed, leading to a SyntaxError. We leave all |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
168 # docstrings as unprefixed to avoid this. This means Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
169 # components touching docstrings need to handle unicode, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
170 # unfortunately. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
171 if s[0:3] in ("'''", '"""'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
172 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
173 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
174 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
175 # If the first character isn't a quote, it is likely a string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
176 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
177 if s[0] not in ("'", '"'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
178 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
179 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
180 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
181 # String literal. Prefix to make a b'' string. |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
182 yield t._replace(string='b%s' % t.string) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
183 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
184 |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
185 # Insert compatibility imports at "from __future__ import" line. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
186 # No '\n' should be added to preserve line numbers. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
187 if (t.type == token.NAME and t.string == 'import' and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
188 all(u.type == token.NAME for u in tokens[i - 2:i]) and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
189 [u.string for u in tokens[i - 2:i]] == ['from', '__future__']): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
190 futureimpline = True |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
191 if t.type == token.NEWLINE and futureimpline: |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
192 futureimpline = False |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
193 if fullname == 'mercurial.pycompat': |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
194 yield t |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
195 continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
196 r, c = t.start |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
197 l = (b'; from mercurial.pycompat import ' |
38785
fb9121ea38c4
py3: stop rewriting xrange() to pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36617
diff
changeset
|
198 b'delattr, getattr, hasattr, setattr, ' |
31843
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
199 b'open, unicode\n') |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
200 for u in tokenize.tokenize(io.BytesIO(l).readline): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
201 if u.type in (tokenize.ENCODING, token.ENDMARKER): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
202 continue |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
203 yield u._replace( |
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
204 start=(r, c + u.start[1]), end=(r, c + u.end[1])) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
205 continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
206 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
207 # This looks like a function call. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
208 if t.type == token.NAME and _isop(i + 1, '('): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
209 fn = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
210 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
211 # *attr() builtins don't accept byte strings to 2nd argument. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
212 if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
213 not _isop(i - 1, '.')): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
214 arg1idx = _findargnofcall(1) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
215 if arg1idx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
216 _ensureunicode(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
217 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
218 # .encode() and .decode() on str/bytes/unicode don't accept |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
219 # byte strings on Python 3. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
220 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
221 for argn in range(2): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
222 argidx = _findargnofcall(argn) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
223 if argidx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
224 _ensureunicode(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
225 |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
226 # It changes iteritems/values to items/values as they are not |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
227 # present in Python 3 world. |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
228 elif fn in ('iteritems', 'itervalues'): |
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
229 yield t._replace(string=fn[4:]) |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
230 continue |
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
231 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
232 # Emit unmodified token. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
233 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
234 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
235 # Header to add to bytecode files. This MUST be changed when |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
236 # ``replacetoken`` or any mechanism that changes semantics of module |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
237 # loading is changed. Otherwise cached bytecode may get loaded without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
238 # the new transformation mechanisms applied. |
38785
fb9121ea38c4
py3: stop rewriting xrange() to pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36617
diff
changeset
|
239 BYTECODEHEADER = b'HG\x00\x0b' |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
240 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
241 class hgloader(importlib.machinery.SourceFileLoader): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
242 """Custom module loader that transforms source code. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
243 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
244 When the source code is converted to a code object, we transform |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
245 certain patterns to be Python 3 compatible. This allows us to write code |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
246 that is natively Python 2 and compatible with Python 3 without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
247 making the code excessively ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
248 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
249 We do this by transforming the token stream between parse and compile. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
250 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
251 Implementing transformations invalidates caching assumptions made |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
252 by the built-in importer. The built-in importer stores a header on |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
253 saved bytecode files indicating the Python/bytecode version. If the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
254 version changes, the cached bytecode is ignored. The Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
255 transformations could change at any time. This means we need to check |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
256 that cached bytecode was generated with the current transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
257 code or there could be a mismatch between cached bytecode and what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
258 would be generated from this class. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
259 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
260 We supplement the bytecode caching layer by wrapping ``get_data`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
261 and ``set_data``. These functions are called when the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
262 ``SourceFileLoader`` retrieves and saves bytecode cache files, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
263 respectively. We simply add an additional header on the file. As |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
264 long as the version in this file is changed when semantics change, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
265 cached bytecode should be invalidated when transformations change. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
266 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
267 The added header has the form ``HG<VERSION>``. That is a literal |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
268 ``HG`` with 2 binary bytes indicating the transformation version. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
269 """ |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
270 def get_data(self, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
271 data = super(hgloader, self).get_data(path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
272 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
273 if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
274 return data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
275 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
276 # There should be a header indicating the Mercurial transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
277 # version. If it doesn't exist or doesn't match the current version, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
278 # we raise an OSError because that is what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
279 # ``SourceFileLoader.get_code()`` expects when loading bytecode |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
280 # paths to indicate the cached file is "bad." |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
281 if data[0:2] != b'HG': |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
282 raise OSError('no hg header') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
283 if data[0:4] != BYTECODEHEADER: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
284 raise OSError('hg header version mismatch') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
285 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
286 return data[4:] |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
287 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
288 def set_data(self, path, data, *args, **kwargs): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
289 if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
290 data = BYTECODEHEADER + data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
291 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
292 return super(hgloader, self).set_data(path, data, *args, **kwargs) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
293 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
294 def source_to_code(self, data, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
295 """Perform token transformation before compilation.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
296 buf = io.BytesIO(data) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
297 tokens = tokenize.tokenize(buf.readline) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
298 data = tokenize.untokenize(replacetokens(list(tokens), self.name)) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
299 # Python's built-in importer strips frames from exceptions raised |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
300 # for this code. Unfortunately, that mechanism isn't extensible |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
301 # and our frame will be blamed for the import failure. There |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
302 # are extremely hacky ways to do frame stripping. We haven't |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
303 # implemented them because they are very ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
304 return super(hgloader, self).source_to_code(data, path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
305 |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
306 # We automagically register our custom importer as a side-effect of |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
307 # loading. This is necessary to ensure that any entry points are able |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
308 # to import mercurial.* modules without having to perform this |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
309 # registration themselves. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
310 if not any(isinstance(x, hgpathentryfinder) for x in sys.meta_path): |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
311 # meta_path is used before any implicit finders and before sys.path. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
312 sys.meta_path.insert(0, hgpathentryfinder()) |