Mercurial > hg
annotate mercurial/__init__.py @ 38618:c829749e7639
shelve: directly handle the initial parent alignment
Shelve is currently sub-contracting some of its work to the rebase extension.
In order to make shelve more independent and flexible we would like shelve to
handle the parent alignment directly.
After this change, we no longer need to use rebase in shelve.
Differential Revision: https://phab.mercurial-scm.org/D3693
author | Boris Feld <boris.feld@octobus.net> |
---|---|
date | Tue, 29 May 2018 00:30:50 +0200 |
parents | 5246f940a48e |
children | fb9121ea38c4 |
rev | line source |
---|---|
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
1 # __init__.py - Startup and module loading logic for Mercurial. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
2 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 from __future__ import absolute_import |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
10 import sys |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
28513
diff
changeset
|
11 |
32420
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
12 # Allow 'from mercurial import demandimport' to keep working. |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
13 import hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
14 demandimport = hgdemandimport |
0906b85bf222
demandimport: move to separate package
Siddharth Agarwal <sid0@fb.com>
parents:
32373
diff
changeset
|
15 |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
16 __all__ = [] |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
17 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
18 # Python 3 uses a custom module loader that transforms source code between |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
19 # source file reading and compilation. This is done by registering a custom |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 # finder that changes the spec for Mercurial modules to use a custom loader. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
21 if sys.version_info[0] >= 3: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
22 import importlib |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
23 import importlib.abc |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
24 import io |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
25 import token |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
26 import tokenize |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
27 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
28 class hgpathentryfinder(importlib.abc.MetaPathFinder): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
29 """A sys.meta_path finder that uses a custom module loader.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
30 def find_spec(self, fullname, path, target=None): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
31 # Only handle Mercurial-related modules. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
32 if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
33 return None |
36617
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
34 # don't try to parse binary |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
35 if fullname.startswith('mercurial.cext.'): |
5246f940a48e
py3: don't try to mangle C extension blob by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
35245
diff
changeset
|
36 return None |
34396
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
37 # third-party packages are expected to be dual-version clean |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
38 if fullname.startswith('mercurial.thirdparty'): |
9fb9f8440b71
python3: don't byte mangle third-party packages
Siddharth Agarwal <sid0@fb.com>
parents:
33502
diff
changeset
|
39 return None |
31307
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
40 # zstd is already dual-version clean, don't try and mangle it |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
41 if fullname.startswith('mercurial.zstd'): |
f8d41edd0357
init: zstd is already python3-ready, so don't run it through our importer
Augie Fackler <raf@durin42.com>
parents:
31150
diff
changeset
|
42 return None |
32521
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
43 # pywatchman is already dual-version clean, don't try and mangle it |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
44 if fullname.startswith('hgext.fsmonitor.pywatchman'): |
942051a29fb6
loader: pywatchman appears to already be py3 compatible
Augie Fackler <raf@durin42.com>
parents:
32425
diff
changeset
|
45 return None |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
46 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
47 # Try to find the module using other registered finders. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
48 spec = None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
49 for finder in sys.meta_path: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
50 if finder == self: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
51 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
52 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
53 spec = finder.find_spec(fullname, path, target=target) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
54 if spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
55 break |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
56 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
57 # This is a Mercurial-related module but we couldn't find it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
58 # using the previously-registered finders. This likely means |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
59 # the module doesn't exist. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
60 if not spec: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
61 return None |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
62 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
63 # TODO need to support loaders from alternate specs, like zip |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
64 # loaders. |
32425
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
65 loader = hgloader(spec.name, spec.origin) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
66 # Can't use util.safehasattr here because that would require |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
67 # importing util, and we're in import code. |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
68 if hasattr(spec.loader, 'loader'): # hasattr-py3-only |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
69 # This is a nested loader (maybe a lazy loader?) |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
70 spec.loader.loader = loader |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
71 else: |
397e3a2e9347
python3: allow hgloader to work with lazy loaders
Siddharth Agarwal <sid0@fb.com>
parents:
32420
diff
changeset
|
72 spec.loader = loader |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
73 return spec |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
74 |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
75 def replacetokens(tokens, fullname): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
76 """Transform a stream of tokens from raw to Python 3. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
77 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
78 It is called by the custom module loading machinery to rewrite |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
79 source/tokens between source decoding and compilation. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
80 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
81 Returns a generator of possibly rewritten tokens. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
82 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
83 The input token list may be mutated as part of processing. However, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
84 its changes do not necessarily match the output token stream. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
85 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
86 REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
87 OR CACHED FILES WON'T GET INVALIDATED PROPERLY. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
88 """ |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
89 futureimpline = False |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
90 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
91 # The following utility functions access the tokens list and i index of |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
92 # the for i, t enumerate(tokens) loop below |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
93 def _isop(j, *o): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
94 """Assert that tokens[j] is an OP with one of the given values""" |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
95 try: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
96 return tokens[j].type == token.OP and tokens[j].string in o |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
97 except IndexError: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
98 return False |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
99 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
100 def _findargnofcall(n): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
101 """Find arg n of a call expression (start at 0) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
102 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
103 Returns index of the first token of that argument, or None if |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
104 there is not that many arguments. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
105 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
106 Assumes that token[i + 1] is '('. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
107 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
108 """ |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
109 nested = 0 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
110 for j in range(i + 2, len(tokens)): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
111 if _isop(j, ')', ']', '}'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
112 # end of call, tuple, subscription or dict / set |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
113 nested -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
114 if nested < 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
115 return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
116 elif n == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
117 # this is the starting position of arg |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
118 return j |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
119 elif _isop(j, '(', '[', '{'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
120 nested += 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
121 elif _isop(j, ',') and nested == 0: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
122 n -= 1 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
123 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
124 return None |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
125 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
126 def _ensureunicode(j): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
127 """Make sure the token at j is a unicode string |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
128 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
129 This rewrites a string token to include the unicode literal prefix |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
130 so the string transformer won't add the byte prefix. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
131 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
132 Ignores tokens that are not strings. Assumes bounds checking has |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
133 already been done. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
134 |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
135 """ |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
136 st = tokens[j] |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
137 if st.type == token.STRING and st.string.startswith(("'", '"')): |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
138 tokens[j] = st._replace(string='u%s' % st.string) |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
139 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
140 for i, t in enumerate(tokens): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
141 # Convert most string literals to byte literals. String literals |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
142 # in Python 2 are bytes. String literals in Python 3 are unicode. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
143 # Most strings in Mercurial are bytes and unicode strings are rare. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
144 # Rather than rewrite all string literals to use ``b''`` to indicate |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
145 # byte strings, we apply this token transformer to insert the ``b`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
146 # prefix nearly everywhere. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
147 if t.type == token.STRING: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
148 s = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
149 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
150 # Preserve docstrings as string literals. This is inconsistent |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
151 # with regular unprefixed strings. However, the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
152 # "from __future__" parsing (which allows a module docstring to |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
153 # exist before it) doesn't properly handle the docstring if it |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
154 # is b''' prefixed, leading to a SyntaxError. We leave all |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
155 # docstrings as unprefixed to avoid this. This means Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
156 # components touching docstrings need to handle unicode, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
157 # unfortunately. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
158 if s[0:3] in ("'''", '"""'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
159 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
160 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
161 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
162 # If the first character isn't a quote, it is likely a string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
163 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
164 if s[0] not in ("'", '"'): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
165 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
166 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
167 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
168 # String literal. Prefix to make a b'' string. |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
169 yield t._replace(string='b%s' % t.string) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
170 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
171 |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
172 # Insert compatibility imports at "from __future__ import" line. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
173 # No '\n' should be added to preserve line numbers. |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
174 if (t.type == token.NAME and t.string == 'import' and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
175 all(u.type == token.NAME for u in tokens[i - 2:i]) and |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
176 [u.string for u in tokens[i - 2:i]] == ['from', '__future__']): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
177 futureimpline = True |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
178 if t.type == token.NEWLINE and futureimpline: |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
179 futureimpline = False |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
180 if fullname == 'mercurial.pycompat': |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
181 yield t |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
182 continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
183 r, c = t.start |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
184 l = (b'; from mercurial.pycompat import ' |
31843
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
185 b'delattr, getattr, hasattr, setattr, xrange, ' |
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
186 b'open, unicode\n') |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
187 for u in tokenize.tokenize(io.BytesIO(l).readline): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
188 if u.type in (tokenize.ENCODING, token.ENDMARKER): |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
189 continue |
30166
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
190 yield u._replace( |
102e6ef5bb3a
py3: use namedtuple._replace to produce new tokens
Martijn Pieters <mjpieters@fb.com>
parents:
30165
diff
changeset
|
191 start=(r, c + u.start[1]), end=(r, c + u.end[1])) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
192 continue |
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
193 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
194 # This looks like a function call. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
195 if t.type == token.NAME and _isop(i + 1, '('): |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
196 fn = t.string |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
197 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
198 # *attr() builtins don't accept byte strings to 2nd argument. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
199 if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
200 not _isop(i - 1, '.')): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
201 arg1idx = _findargnofcall(1) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
202 if arg1idx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
203 _ensureunicode(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
205 # .encode() and .decode() on str/bytes/unicode don't accept |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
206 # byte strings on Python 3. |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
207 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
208 for argn in range(2): |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
209 argidx = _findargnofcall(argn) |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
210 if argidx is not None: |
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
211 _ensureunicode(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
212 |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
213 # It changes iteritems/values to items/values as they are not |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
214 # present in Python 3 world. |
31445
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
215 elif fn in ('iteritems', 'itervalues'): |
83e080144faf
py3: rewrite itervalues() as values() by importer
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
216 yield t._replace(string=fn[4:]) |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
217 continue |
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
218 |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
219 # Emit unmodified token. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
220 yield t |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
221 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
222 # Header to add to bytecode files. This MUST be changed when |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
223 # ``replacetoken`` or any mechanism that changes semantics of module |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
224 # loading is changed. Otherwise cached bytecode may get loaded without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
225 # the new transformation mechanisms applied. |
31843
526e4597cca5
py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31445
diff
changeset
|
226 BYTECODEHEADER = b'HG\x00\x0a' |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
227 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
228 class hgloader(importlib.machinery.SourceFileLoader): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
229 """Custom module loader that transforms source code. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
230 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
231 When the source code is converted to a code object, we transform |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
232 certain patterns to be Python 3 compatible. This allows us to write code |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
233 that is natively Python 2 and compatible with Python 3 without |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
234 making the code excessively ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
235 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
236 We do this by transforming the token stream between parse and compile. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
237 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
238 Implementing transformations invalidates caching assumptions made |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
239 by the built-in importer. The built-in importer stores a header on |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
240 saved bytecode files indicating the Python/bytecode version. If the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
241 version changes, the cached bytecode is ignored. The Mercurial |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
242 transformations could change at any time. This means we need to check |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
243 that cached bytecode was generated with the current transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
244 code or there could be a mismatch between cached bytecode and what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
245 would be generated from this class. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
246 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
247 We supplement the bytecode caching layer by wrapping ``get_data`` |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
248 and ``set_data``. These functions are called when the |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
249 ``SourceFileLoader`` retrieves and saves bytecode cache files, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
250 respectively. We simply add an additional header on the file. As |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
251 long as the version in this file is changed when semantics change, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
252 cached bytecode should be invalidated when transformations change. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
253 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
254 The added header has the form ``HG<VERSION>``. That is a literal |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
255 ``HG`` with 2 binary bytes indicating the transformation version. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
256 """ |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
257 def get_data(self, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
258 data = super(hgloader, self).get_data(path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
259 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
260 if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
261 return data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
262 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
263 # There should be a header indicating the Mercurial transformation |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
264 # version. If it doesn't exist or doesn't match the current version, |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
265 # we raise an OSError because that is what |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
266 # ``SourceFileLoader.get_code()`` expects when loading bytecode |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
267 # paths to indicate the cached file is "bad." |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
268 if data[0:2] != b'HG': |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
269 raise OSError('no hg header') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
270 if data[0:4] != BYTECODEHEADER: |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
271 raise OSError('hg header version mismatch') |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
272 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
273 return data[4:] |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
274 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
275 def set_data(self, path, data, *args, **kwargs): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
276 if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
277 data = BYTECODEHEADER + data |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
278 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
279 return super(hgloader, self).set_data(path, data, *args, **kwargs) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
280 |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
281 def source_to_code(self, data, path): |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
282 """Perform token transformation before compilation.""" |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
283 buf = io.BytesIO(data) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
284 tokens = tokenize.tokenize(buf.readline) |
29800
178c89e8519a
py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents:
29550
diff
changeset
|
285 data = tokenize.untokenize(replacetokens(list(tokens), self.name)) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
286 # Python's built-in importer strips frames from exceptions raised |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
287 # for this code. Unfortunately, that mechanism isn't extensible |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
288 # and our frame will be blamed for the import failure. There |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
289 # are extremely hacky ways to do frame stripping. We haven't |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
290 # implemented them because they are very ugly. |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
291 return super(hgloader, self).source_to_code(data, path) |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
292 |
32373
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
293 # We automagically register our custom importer as a side-effect of |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
294 # loading. This is necessary to ensure that any entry points are able |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
295 # to import mercurial.* modules without having to perform this |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
296 # registration themselves. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
297 if not any(isinstance(x, hgpathentryfinder) for x in sys.meta_path): |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
298 # meta_path is used before any implicit finders and before sys.path. |
5700825889fb
policy: drop custom importer for pure modules
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
299 sys.meta_path.insert(0, hgpathentryfinder()) |