Mercurial > hg
annotate mercurial/__init__.py @ 38785:fb9121ea38c4
py3: stop rewriting xrange() to pycompat.xrange()
We now require the use of pycompat.xrange() in source.
One less feature in the module importer gets us one step closer
to Python 3 native source code.
Differential Revision: https://phab.mercurial-scm.org/D4034
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Wed, 01 Aug 2018 13:10:07 -0700 |
parents | 5246f940a48e |
children | 4277e20cfec4 |
rev | line source |
---|---|
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
1 # __init__.py - Startup and module loading logic for Mercurial. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
2 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 from __future__ import absolute_import |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
10 import sys |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
28513
diff
changeset
|
11 |
32420
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
12 # Allow 'from mercurial import demandimport' to keep working. |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
13 import hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
14 demandimport = hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
15 |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
16 __all__ = [] |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
17 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
18 # Python 3 uses a custom module loader that transforms source code between |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
19 # source file reading and compilation. This is done by registering a custom |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 # finder that changes the spec for Mercurial modules to use a custom loader. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
21 if sys.version_info[0] >= 3: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
22 import importlib |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
23 import importlib.abc |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
24 import io |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
25 import token |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
26 import tokenize |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
27 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
28 class hgpathentryfinder(importlib.abc.MetaPathFinder): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
29 """A sys.meta_path finder that uses a custom module loader.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
30 def find_spec(self, fullname, path, target=None): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
31 # Only handle Mercurial-related modules. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
32 if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
33 return None |
36617
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
34 # don't try to parse binary |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
35 if fullname.startswith('mercurial.cext.'): |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
36 return None |
34396
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
37 # third-party packages are expected to be dual-version clean |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
38 if fullname.startswith('mercurial.thirdparty'): |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
39 return None |
31307
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
40 # zstd is already dual-version clean, don't try and mangle it |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
41 if fullname.startswith('mercurial.zstd'): |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
42 return None |
32521
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
43 # pywatchman is already dual-version clean, don't try and mangle it |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
44 if fullname.startswith('hgext.fsmonitor.pywatchman'): |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
45 return None |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
46 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
47 # Try to find the module using other registered finders. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
48 spec = None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
49 for finder in sys.meta_path: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
50 if finder == self: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
51 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
52 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
53 spec = finder.find_spec(fullname, path, target=target) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
54 if spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
55 break |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
56 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
57 # This is a Mercurial-related module but we couldn't find it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
58 # using the previously-registered finders. This likely means |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
59 # the module doesn't exist. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
60 if not spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
61 return None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
62 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
63 # TODO need to support loaders from alternate specs, like zip |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
64 # loaders. |
32425
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
65 loader = hgloader(spec.name, spec.origin) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
66 # Can't use util.safehasattr here because that would require |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
67 # importing util, and we're in import code. |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
68 if hasattr(spec.loader, 'loader'): # hasattr-py3-only |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
69 # This is a nested loader (maybe a lazy loader?) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
70 spec.loader.loader = loader |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
71 else: |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
72 spec.loader = loader |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
73 return spec |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
74 |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
75 def replacetokens(tokens, fullname): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
76 """Transform a stream of tokens from raw to Python 3. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
77 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
78 It is called by the custom module loading machinery to rewrite |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
79 source/tokens between source decoding and compilation. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
80 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
81 Returns a generator of possibly rewritten tokens. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
82 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
83 The input token list may be mutated as part of processing. However, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
84 its changes do not necessarily match the output token stream. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
85 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
86 REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
87 OR CACHED FILES WON'T GET INVALIDATED PROPERLY. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
88 """ |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
89 futureimpline = False |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
90 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
91 # The following utility functions access the tokens list and i index of |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
92 # the for i, t enumerate(tokens) loop below |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
93 def _isop(j, *o): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
94 """Assert that tokens[j] is an OP with one of the given values""" |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
95 try: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
96 return tokens[j].type == token.OP and tokens[j].string in o |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
97 except IndexError: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
98 return False |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
99 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
100 def _findargnofcall(n): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
101 """Find arg n of a call expression (start at 0) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
102 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
103 Returns index of the first token of that argument, or None if |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
104 there is not that many arguments. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
105 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
106 Assumes that token[i + 1] is '('. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
107 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
108 """ |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
109 nested = 0 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
110 for j in range(i + 2, len(tokens)): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
111 if _isop(j, ')', ']', '}'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
112 # end of call, tuple, subscription or dict / set |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
113 nested -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
114 if nested < 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
115 return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
116 elif n == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
117 # this is the starting position of arg |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
118 return j |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
119 elif _isop(j, '(', '[', '{'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
120 nested += 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
121 elif _isop(j, ',') and nested == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
122 n -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
123 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
124 return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
125 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
126 def _ensureunicode(j): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
127 """Make sure the token at j is a unicode string |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
128 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
129 This rewrites a string token to include the unicode literal prefix |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
130 so the string transformer won't add the byte prefix. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
131 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
132 Ignores tokens that are not strings. Assumes bounds checking has |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
133 already been done. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
134 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
135 """ |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
136 st = tokens[j] |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
137 if st.type == token.STRING and st.string.startswith(("'", '"')): |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
138 tokens[j] = st._replace(string='u%s' % st.string) |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
139 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
140 for i, t in enumerate(tokens): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
141 # Convert most string literals to byte literals. String literals |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
142 # in Python 2 are bytes. String literals in Python 3 are unicode. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
143 # Most strings in Mercurial are bytes and unicode strings are rare. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
144 # Rather than rewrite all string literals to use ``b''`` to indicate |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
145 # byte strings, we apply this token transformer to insert the ``b`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
146 # prefix nearly everywhere. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
147 if t.type == token.STRING: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
148 s = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
149 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
150 # Preserve docstrings as string literals. This is inconsistent |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
151 # with regular unprefixed strings. However, the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
152 # "from __future__" parsing (which allows a module docstring to |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
153 # exist before it) doesn't properly handle the docstring if it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
154 # is b''' prefixed, leading to a SyntaxError. We leave all |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
155 # docstrings as unprefixed to avoid this. This means Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
156 # components touching docstrings need to handle unicode, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
157 # unfortunately. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
158 if s[0:3] in ("'''", '"""'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
159 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
160 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
161 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
162 # If the first character isn't a quote, it is likely a string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
163 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
164 if s[0] not in ("'", '"'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
165 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
166 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
167 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
168 # String literal. Prefix to make a b'' string. |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
169 yield t._replace(string='b%s' % t.string) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
170 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
171 |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
172 # Insert compatibility imports at "from __future__ import" line. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
173 # No '\n' should be added to preserve line numbers. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
174 if (t.type == token.NAME and t.string == 'import' and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
175 all(u.type == token.NAME for u in tokens[i - 2:i]) and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
176 [u.string for u in tokens[i - 2:i]] == ['from', '__future__']): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
177 futureimpline = True |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
178 if t.type == token.NEWLINE and futureimpline: |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
179 futureimpline = False |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
180 if fullname == 'mercurial.pycompat': |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
181 yield t |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
182 continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
183 r, c = t.start |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
184 l = (b'; from mercurial.pycompat import ' |
38785
fb9121ea38c4
py3: stop rewriting xrange() to pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36617
diff
changeset
|
185 b'delattr, getattr, hasattr, setattr, ' |
31843
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
186 b'open, unicode\n') |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
187 for u in tokenize.tokenize(io.BytesIO(l).readline): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
188 if u.type in (tokenize.ENCODING, token.ENDMARKER): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
189 continue |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
190 yield u._replace( |
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
191 start=(r, c + u.start[1]), end=(r, c + u.end[1])) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
192 continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
193 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
194 # This looks like a function call. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
195 if t.type == token.NAME and _isop(i + 1, '('): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
196 fn = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
197 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
198 # *attr() builtins don't accept byte strings to 2nd argument. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
199 if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
200 not _isop(i - 1, '.')): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
201 arg1idx = _findargnofcall(1) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
202 if arg1idx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
203 _ensureunicode(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
205 # .encode() and .decode() on str/bytes/unicode don't accept |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
206 # byte strings on Python 3. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
207 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
208 for argn in range(2): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
209 argidx = _findargnofcall(argn) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
210 if argidx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
211 _ensureunicode(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
212 |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
213 # It changes iteritems/values to items/values as they are not |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
214 # present in Python 3 world. |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
215 elif fn in ('iteritems', 'itervalues'): |
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
216 yield t._replace(string=fn[4:]) |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
217 continue |
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
218 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
219 # Emit unmodified token. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
220 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
221 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
222 # Header to add to bytecode files. This MUST be changed when |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
223 # ``replacetoken`` or any mechanism that changes semantics of module |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
224 # loading is changed. Otherwise cached bytecode may get loaded without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
225 # the new transformation mechanisms applied. |
38785
fb9121ea38c4
py3: stop rewriting xrange() to pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36617
diff
changeset
|
226 BYTECODEHEADER = b'HG\x00\x0b' |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
227 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
228 class hgloader(importlib.machinery.SourceFileLoader): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
229 """Custom module loader that transforms source code. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
230 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
231 When the source code is converted to a code object, we transform |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
232 certain patterns to be Python 3 compatible. This allows us to write code |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
233 that is natively Python 2 and compatible with Python 3 without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
234 making the code excessively ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
235 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
236 We do this by transforming the token stream between parse and compile. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
237 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
238 Implementing transformations invalidates caching assumptions made |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
239 by the built-in importer. The built-in importer stores a header on |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
240 saved bytecode files indicating the Python/bytecode version. If the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
241 version changes, the cached bytecode is ignored. The Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
242 transformations could change at any time. This means we need to check |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
243 that cached bytecode was generated with the current transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
244 code or there could be a mismatch between cached bytecode and what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
245 would be generated from this class. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
246 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
247 We supplement the bytecode caching layer by wrapping ``get_data`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
248 and ``set_data``. These functions are called when the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
249 ``SourceFileLoader`` retrieves and saves bytecode cache files, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
250 respectively. We simply add an additional header on the file. As |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
251 long as the version in this file is changed when semantics change, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
252 cached bytecode should be invalidated when transformations change. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
253 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
254 The added header has the form ``HG<VERSION>``. That is a literal |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
255 ``HG`` with 2 binary bytes indicating the transformation version. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
256 """ |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
257 def get_data(self, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
258 data = super(hgloader, self).get_data(path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
259 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
260 if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
261 return data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
262 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
263 # There should be a header indicating the Mercurial transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
264 # version. If it doesn't exist or doesn't match the current version, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
265 # we raise an OSError because that is what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
266 # ``SourceFileLoader.get_code()`` expects when loading bytecode |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
267 # paths to indicate the cached file is "bad." |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
268 if data[0:2] != b'HG': |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
269 raise OSError('no hg header') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
270 if data[0:4] != BYTECODEHEADER: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
271 raise OSError('hg header version mismatch') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
272 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
273 return data[4:] |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
274 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
275 def set_data(self, path, data, *args, **kwargs): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
276 if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
277 data = BYTECODEHEADER + data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
278 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
279 return super(hgloader, self).set_data(path, data, *args, **kwargs) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
280 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
281 def source_to_code(self, data, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
282 """Perform token transformation before compilation.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
283 buf = io.BytesIO(data) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
284 tokens = tokenize.tokenize(buf.readline) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
285 data = tokenize.untokenize(replacetokens(list(tokens), self.name)) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
286 # Python's built-in importer strips frames from exceptions raised |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
287 # for this code. Unfortunately, that mechanism isn't extensible |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
288 # and our frame will be blamed for the import failure. There |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
289 # are extremely hacky ways to do frame stripping. We haven't |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
290 # implemented them because they are very ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
291 return super(hgloader, self).source_to_code(data, path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
292 |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
293 # We automagically register our custom importer as a side-effect of |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
294 # loading. This is necessary to ensure that any entry points are able |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
295 # to import mercurial.* modules without having to perform this |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
296 # registration themselves. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
297 if not any(isinstance(x, hgpathentryfinder) for x in sys.meta_path): |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
298 # meta_path is used before any implicit finders and before sys.path. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
299 sys.meta_path.insert(0, hgpathentryfinder()) |