Mercurial > hg
annotate contrib/byteify-strings.py @ 44528:c8891bca40fb
rust-status: add bare `hg status` support in hg-core
A lot of performance remains to be gained, most notably by doing more things
in parallel, but also by caching, not falling back to Python but switching
to another regex engine, etc..
I have measured on multiple repositories that this change, when in combination
with the next two patches, improve bare `hg status` performance, and has no
observable impact when falling back (because it does so early).
On the Netbeans repository:
C: 840ms
Rust+C: 556ms
Mozilla Central with the one pattern that causes a fallback removed:
C: 2.315s
Rust+C: 1.700 s
Differential Revision: https://phab.mercurial-scm.org/D7929
author | Raphaël Gomès <rgomes@octobus.net> |
---|---|
date | Fri, 17 Jan 2020 15:43:46 +0100 |
parents | 09234eeae206 |
children | bc9a9016467d |
rev | line source |
---|---|
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
1 #!/usr/bin/env python3 |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
2 # |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
3 # byteify-strings.py - transform string literals to be Python 3 safe |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
10 from __future__ import absolute_import, print_function |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
11 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
12 import argparse |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
13 import contextlib |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
14 import errno |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
15 import os |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
16 import sys |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
17 import tempfile |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
18 import token |
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
19 import tokenize |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
21 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
22 def adjusttokenpos(t, ofs): |
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
23 """Adjust start/end column of the given token""" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
24 return t._replace( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
25 start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
26 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
27 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
28 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
29 def replacetokens(tokens, opts): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
30 """Transform a stream of tokens from raw to Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
31 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
32 Returns a generator of possibly rewritten tokens. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
33 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
34 The input token list may be mutated as part of processing. However, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
35 its changes do not necessarily match the output token stream. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
36 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
37 sysstrtokens = set() |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
38 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
39 # The following utility functions access the tokens list and i index of |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
40 # the for i, t enumerate(tokens) loop below |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
41 def _isop(j, *o): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
42 """Assert that tokens[j] is an OP with one of the given values""" |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
43 try: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
44 return tokens[j].type == token.OP and tokens[j].string in o |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
45 except IndexError: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
46 return False |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
47 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
48 def _findargnofcall(n): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
49 """Find arg n of a call expression (start at 0) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
50 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
51 Returns index of the first token of that argument, or None if |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
52 there is not that many arguments. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
53 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
54 Assumes that token[i + 1] is '('. |
38389
1d68fd5f614a
byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents:
38388
diff
changeset
|
55 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
56 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
57 nested = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
58 for j in range(i + 2, len(tokens)): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
59 if _isop(j, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
60 # end of call, tuple, subscription or dict / set |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
61 nested -= 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
62 if nested < 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
63 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
64 elif n == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
65 # this is the starting position of arg |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
66 return j |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
67 elif _isop(j, '(', '[', '{'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
68 nested += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
69 elif _isop(j, ',') and nested == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
70 n -= 1 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
71 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
72 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
73 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
74 def _ensuresysstr(j): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
75 """Make sure the token at j is a system string |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
76 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
77 Remember the given token so the string transformer won't add |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
78 the byte prefix. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
79 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
80 Ignores tokens that are not strings. Assumes bounds checking has |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
81 already been done. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
82 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
83 """ |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
84 k = j |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
85 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
86 while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
87 k += 1 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
88 if currtoken.type == token.STRING and currtoken.string.startswith( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
89 ("'", '"') |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
90 ): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
91 sysstrtokens.add(currtoken) |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
92 try: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
93 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
94 except IndexError: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
95 break |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
96 |
42677
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
97 def _isitemaccess(j): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
98 """Assert the next tokens form an item access on `tokens[j]` and that |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
99 `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
100 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
101 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
102 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
103 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
104 and _isop(j + 1, '[') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
105 and tokens[j + 2].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
106 and _isop(j + 3, ']') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
107 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
108 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
109 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
110 |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
111 def _ismethodcall(j, *methodnames): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
112 """Assert the next tokens form a call to `methodname` with a string |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
113 as first argument on `tokens[j]` and that `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
114 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
115 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
116 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
117 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
118 and _isop(j + 1, '.') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
119 and tokens[j + 2].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
120 and tokens[j + 2].string in methodnames |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
121 and _isop(j + 3, '(') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
122 and tokens[j + 4].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
123 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
124 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
125 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
126 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
127 coldelta = 0 # column increment for new opening parens |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
128 coloffset = -1 # column offset for the current line (-1: TBD) |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
129 parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
130 ignorenextline = False # don't transform the next line |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
131 insideignoreblock = False # don't transform until turned off |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
132 for i, t in enumerate(tokens): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
133 # Compute the column offset for the current line, such that |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
134 # the current line will be aligned to the last opening paren |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
135 # as before. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
136 if coloffset < 0: |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
137 lastparen = parens[-1] |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
138 if t.start[1] == lastparen[1]: |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
139 coloffset = lastparen[2] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
140 elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
141 token.NEWLINE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
142 tokenize.NL, |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
143 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
144 # fix misaligned indent of s/util.Abort/error.Abort/ |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
145 coloffset = lastparen[2] + (lastparen[1] - t.start[1]) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
146 else: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
147 coloffset = 0 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
148 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
149 # Reset per-line attributes at EOL. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
150 if t.type in (token.NEWLINE, tokenize.NL): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
151 yield adjusttokenpos(t, coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
152 coldelta = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
153 coloffset = -1 |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
154 if not insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
155 ignorenextline = ( |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
156 tokens[i - 1].type == token.COMMENT |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
157 and tokens[i - 1].string == "# no-py3-transform" |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
158 ) |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
159 continue |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
160 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
161 if t.type == token.COMMENT: |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
162 if t.string == "# py3-transform: off": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
163 insideignoreblock = True |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
164 if t.string == "# py3-transform: on": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
165 insideignoreblock = False |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
166 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
167 if ignorenextline or insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
168 yield adjusttokenpos(t, coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
169 continue |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
170 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
171 # Remember the last paren position. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
172 if _isop(i, '(', '[', '{'): |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
173 parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type)) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
174 elif _isop(i, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
175 parens.pop() |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
176 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
177 # Convert most string literals to byte literals. String literals |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
178 # in Python 2 are bytes. String literals in Python 3 are unicode. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
179 # Most strings in Mercurial are bytes and unicode strings are rare. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
180 # Rather than rewrite all string literals to use ``b''`` to indicate |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
181 # byte strings, we apply this token transformer to insert the ``b`` |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
182 # prefix nearly everywhere. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
183 if t.type == token.STRING and t not in sysstrtokens: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
184 s = t.string |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
185 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
186 # Preserve docstrings as string literals. This is inconsistent |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
187 # with regular unprefixed strings. However, the |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
188 # "from __future__" parsing (which allows a module docstring to |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
189 # exist before it) doesn't properly handle the docstring if it |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
190 # is b''' prefixed, leading to a SyntaxError. We leave all |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
191 # docstrings as unprefixed to avoid this. This means Mercurial |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
192 # components touching docstrings need to handle unicode, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
193 # unfortunately. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
194 if s[0:3] in ("'''", '"""'): |
42675
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
195 # If it's assigned to something, it's not a docstring |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
196 if not _isop(i - 1, '='): |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
197 yield adjusttokenpos(t, coloffset) |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
198 continue |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
199 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
200 # If the first character isn't a quote, it is likely a string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
201 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
202 if s[0] not in ("'", '"'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
203 yield adjusttokenpos(t, coloffset) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
205 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
206 # String literal. Prefix to make a b'' string. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
207 yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
208 coldelta += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
209 continue |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
210 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
211 # This looks like a function call. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
212 if t.type == token.NAME and _isop(i + 1, '('): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
213 fn = t.string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
214 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
215 # *attr() builtins don't accept byte strings to 2nd argument. |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
216 if fn in ( |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
217 'getattr', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
218 'setattr', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
219 'hasattr', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
220 'safehasattr', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
221 'wrapfunction', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
222 'wrapclass', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
223 'addattr', |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
224 ) and (opts['allow-attr-methods'] or not _isop(i - 1, '.')): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
225 arg1idx = _findargnofcall(1) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
226 if arg1idx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
227 _ensuresysstr(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
228 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
229 # .encode() and .decode() on str/bytes/unicode don't accept |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
230 # byte strings on Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
231 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
232 for argn in range(2): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
233 argidx = _findargnofcall(argn) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
234 if argidx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
235 _ensuresysstr(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
236 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
237 # It changes iteritems/values to items/values as they are not |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
238 # present in Python 3 world. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
239 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
240 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
241 continue |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
242 |
42678
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
243 if t.type == token.NAME and t.string in opts['treat-as-kwargs']: |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
244 if _isitemaccess(i): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
245 _ensuresysstr(i + 2) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
246 if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
247 _ensuresysstr(i + 4) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
248 |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
249 # Looks like "if __name__ == '__main__'". |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
250 if ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
251 t.type == token.NAME |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
252 and t.string == '__name__' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
253 and _isop(i + 1, '==') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
254 ): |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
255 _ensuresysstr(i + 2) |
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
256 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
257 # Emit unmodified token. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
258 yield adjusttokenpos(t, coloffset) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
259 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
260 |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
261 def process(fin, fout, opts): |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
262 tokens = tokenize.tokenize(fin.readline) |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
263 tokens = replacetokens(list(tokens), opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
264 fout.write(tokenize.untokenize(tokens)) |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
265 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
266 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
267 def tryunlink(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
268 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
269 os.unlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
270 except OSError as err: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
271 if err.errno != errno.ENOENT: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
272 raise |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
273 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
274 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
275 @contextlib.contextmanager |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
276 def editinplace(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
277 n = os.path.basename(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
278 d = os.path.dirname(fname) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
279 fp = tempfile.NamedTemporaryFile( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
280 prefix='.%s-' % n, suffix='~', dir=d, delete=False |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
281 ) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
282 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
283 yield fp |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
284 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
285 if os.name == 'nt': |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
286 tryunlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
287 os.rename(fp.name, fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
288 finally: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
289 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
290 tryunlink(fp.name) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
291 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
292 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
293 def main(): |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
294 ap = argparse.ArgumentParser() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
295 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
296 '--version', action='version', version='Byteify strings 1.0' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
297 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
298 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
299 '-i', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
300 '--inplace', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
301 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
302 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
303 help='edit files in place', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
304 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
305 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
306 '--dictiter', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
307 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
308 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
309 help='rewrite iteritems() and itervalues()', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
310 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
311 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
312 '--allow-attr-methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
313 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
314 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
315 help='also handle attr*() when they are methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
316 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
317 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
318 '--treat-as-kwargs', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
319 nargs="+", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
320 default=[], |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
321 help="ignore kwargs-like objects", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
322 ), |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
323 ap.add_argument('files', metavar='FILE', nargs='+', help='source file') |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
324 args = ap.parse_args() |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
325 opts = { |
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
326 'dictiter': args.dictiter, |
42679
5e296f618920
byteify-strings: simplify default value for `--treat-as-kwargs`
Raphaël Gomès <rgomes@octobus.net>
parents:
42678
diff
changeset
|
327 'treat-as-kwargs': set(args.treat_as_kwargs), |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
328 'allow-attr-methods': args.allow_attr_methods, |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
329 } |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
330 for fname in args.files: |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
331 if args.inplace: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
332 with editinplace(fname) as fout: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
333 with open(fname, 'rb') as fin: |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
334 process(fin, fout, opts) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
335 else: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
336 with open(fname, 'rb') as fin: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
337 fout = sys.stdout.buffer |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
338 process(fin, fout, opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
339 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
340 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
341 if __name__ == '__main__': |
43407
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
342 if sys.version_info[0:2] < (3, 7): |
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
343 print('This script must be run under Python 3.7+') |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
344 sys.exit(3) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
345 main() |