Mercurial > hg
annotate contrib/byteify-strings.py @ 49370:1572f790ee5e stable
convert: remove old ElementTree import cruft from darcs
All the `import elementtree` attempts seem to pre-date py2.5, when it was
brought into the standard library, and the manual `cElementTree` fast
implementation import has been unnecessary and deprecated since py3.3.
author | Ian Moody <moz-ian@perix.co.uk> |
---|---|
date | Wed, 15 Jun 2022 01:01:02 +0100 |
parents | 6000f5b25c9b |
children | 8250ecb53f30 |
rev | line source |
---|---|
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
1 #!/usr/bin/env python3 |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
2 # |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
3 # byteify-strings.py - transform string literals to be Python 3 safe |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
10 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
11 import argparse |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
12 import contextlib |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
13 import errno |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
14 import os |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
15 import sys |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
16 import tempfile |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
17 import token |
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
18 import tokenize |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
19 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
20 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
21 def adjusttokenpos(t, ofs): |
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
22 """Adjust start/end column of the given token""" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
23 return t._replace( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
24 start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
25 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
26 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
27 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
28 def replacetokens(tokens, opts): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
29 """Transform a stream of tokens from raw to Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
30 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
31 Returns a generator of possibly rewritten tokens. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
32 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
33 The input token list may be mutated as part of processing. However, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
34 its changes do not necessarily match the output token stream. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
35 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
36 sysstrtokens = set() |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
37 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
38 # The following utility functions access the tokens list and i index of |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
39 # the for i, t enumerate(tokens) loop below |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
40 def _isop(j, *o): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
41 """Assert that tokens[j] is an OP with one of the given values""" |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
42 try: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
43 return tokens[j].type == token.OP and tokens[j].string in o |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
44 except IndexError: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
45 return False |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
46 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
47 def _findargnofcall(n): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
48 """Find arg n of a call expression (start at 0) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
49 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
50 Returns index of the first token of that argument, or None if |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
51 there is not that many arguments. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
52 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
53 Assumes that token[i + 1] is '('. |
38389
1d68fd5f614a
byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents:
38388
diff
changeset
|
54 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
55 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
56 nested = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
57 for j in range(i + 2, len(tokens)): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
58 if _isop(j, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
59 # end of call, tuple, subscription or dict / set |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
60 nested -= 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
61 if nested < 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
62 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
63 elif n == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
64 # this is the starting position of arg |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
65 return j |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
66 elif _isop(j, '(', '[', '{'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
67 nested += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
68 elif _isop(j, ',') and nested == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
69 n -= 1 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
70 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
71 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
72 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
73 def _ensuresysstr(j): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
74 """Make sure the token at j is a system string |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
75 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
76 Remember the given token so the string transformer won't add |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
77 the byte prefix. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
78 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
79 Ignores tokens that are not strings. Assumes bounds checking has |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
80 already been done. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
81 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
82 """ |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
83 k = j |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
84 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
85 while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
86 k += 1 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
87 if currtoken.type == token.STRING and currtoken.string.startswith( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
88 ("'", '"') |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
89 ): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
90 sysstrtokens.add(currtoken) |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
91 try: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
92 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
93 except IndexError: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
94 break |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
95 |
42677
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
96 def _isitemaccess(j): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
97 """Assert the next tokens form an item access on `tokens[j]` and that |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
98 `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
99 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
100 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
101 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
102 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
103 and _isop(j + 1, '[') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
104 and tokens[j + 2].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
105 and _isop(j + 3, ']') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
106 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
107 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
108 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
109 |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
110 def _ismethodcall(j, *methodnames): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
111 """Assert the next tokens form a call to `methodname` with a string |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
112 as first argument on `tokens[j]` and that `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
113 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
114 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
115 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
116 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
117 and _isop(j + 1, '.') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
118 and tokens[j + 2].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
119 and tokens[j + 2].string in methodnames |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
120 and _isop(j + 3, '(') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
121 and tokens[j + 4].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
122 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
123 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
124 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
125 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
126 coldelta = 0 # column increment for new opening parens |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
127 coloffset = -1 # column offset for the current line (-1: TBD) |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
128 parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
129 ignorenextline = False # don't transform the next line |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
130 insideignoreblock = False # don't transform until turned off |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
131 for i, t in enumerate(tokens): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
132 # Compute the column offset for the current line, such that |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
133 # the current line will be aligned to the last opening paren |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
134 # as before. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
135 if coloffset < 0: |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
136 lastparen = parens[-1] |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
137 if t.start[1] == lastparen[1]: |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
138 coloffset = lastparen[2] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
139 elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
140 token.NEWLINE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
141 tokenize.NL, |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
142 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
143 # fix misaligned indent of s/util.Abort/error.Abort/ |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
144 coloffset = lastparen[2] + (lastparen[1] - t.start[1]) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
145 else: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
146 coloffset = 0 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
147 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
148 # Reset per-line attributes at EOL. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
149 if t.type in (token.NEWLINE, tokenize.NL): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
150 yield adjusttokenpos(t, coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
151 coldelta = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
152 coloffset = -1 |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
153 if not insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
154 ignorenextline = ( |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
155 tokens[i - 1].type == token.COMMENT |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
156 and tokens[i - 1].string == "# no-py3-transform" |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
157 ) |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
158 continue |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
159 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
160 if t.type == token.COMMENT: |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
161 if t.string == "# py3-transform: off": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
162 insideignoreblock = True |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
163 if t.string == "# py3-transform: on": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
164 insideignoreblock = False |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
165 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
166 if ignorenextline or insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
167 yield adjusttokenpos(t, coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
168 continue |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
169 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
170 # Remember the last paren position. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
171 if _isop(i, '(', '[', '{'): |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
172 parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type)) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
173 elif _isop(i, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
174 parens.pop() |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
175 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
176 # Convert most string literals to byte literals. String literals |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
177 # in Python 2 are bytes. String literals in Python 3 are unicode. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
178 # Most strings in Mercurial are bytes and unicode strings are rare. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
179 # Rather than rewrite all string literals to use ``b''`` to indicate |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
180 # byte strings, we apply this token transformer to insert the ``b`` |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
181 # prefix nearly everywhere. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
182 if t.type == token.STRING and t not in sysstrtokens: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
183 s = t.string |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
184 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
185 # Preserve docstrings as string literals. This is inconsistent |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
186 # with regular unprefixed strings. However, the |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
187 # "from __future__" parsing (which allows a module docstring to |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
188 # exist before it) doesn't properly handle the docstring if it |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
189 # is b''' prefixed, leading to a SyntaxError. We leave all |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
190 # docstrings as unprefixed to avoid this. This means Mercurial |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
191 # components touching docstrings need to handle unicode, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
192 # unfortunately. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
193 if s[0:3] in ("'''", '"""'): |
42675
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
194 # If it's assigned to something, it's not a docstring |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
195 if not _isop(i - 1, '='): |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
196 yield adjusttokenpos(t, coloffset) |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
197 continue |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
198 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
199 # If the first character isn't a quote, it is likely a string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
200 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
201 if s[0] not in ("'", '"'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
202 yield adjusttokenpos(t, coloffset) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
203 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
205 # String literal. Prefix to make a b'' string. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
206 yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
207 coldelta += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
208 continue |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
209 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
210 # This looks like a function call. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
211 if t.type == token.NAME and _isop(i + 1, '('): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
212 fn = t.string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
213 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
214 # *attr() builtins don't accept byte strings to 2nd argument. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
215 if ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
216 fn |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
217 in ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
218 'getattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
219 'setattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
220 'hasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
221 'safehasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
222 'wrapfunction', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
223 'wrapclass', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
224 'addattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
225 ) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
226 and (opts['allow-attr-methods'] or not _isop(i - 1, '.')) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
227 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
228 arg1idx = _findargnofcall(1) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
229 if arg1idx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
230 _ensuresysstr(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
231 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
232 # .encode() and .decode() on str/bytes/unicode don't accept |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
233 # byte strings on Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
234 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
235 for argn in range(2): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
236 argidx = _findargnofcall(argn) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
237 if argidx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
238 _ensuresysstr(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
239 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
240 # It changes iteritems/values to items/values as they are not |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
241 # present in Python 3 world. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
242 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
243 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
244 continue |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
245 |
42678
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
246 if t.type == token.NAME and t.string in opts['treat-as-kwargs']: |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
247 if _isitemaccess(i): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
248 _ensuresysstr(i + 2) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
249 if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
250 _ensuresysstr(i + 4) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
251 |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
252 # Looks like "if __name__ == '__main__'". |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
253 if ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
254 t.type == token.NAME |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
255 and t.string == '__name__' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
256 and _isop(i + 1, '==') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
257 ): |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
258 _ensuresysstr(i + 2) |
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
259 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
260 # Emit unmodified token. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
261 yield adjusttokenpos(t, coloffset) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
262 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
263 |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
264 def process(fin, fout, opts): |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
265 tokens = tokenize.tokenize(fin.readline) |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
266 tokens = replacetokens(list(tokens), opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
267 fout.write(tokenize.untokenize(tokens)) |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
268 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
269 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
270 def tryunlink(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
271 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
272 os.unlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
273 except OSError as err: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
274 if err.errno != errno.ENOENT: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
275 raise |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
276 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
277 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
278 @contextlib.contextmanager |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
279 def editinplace(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
280 n = os.path.basename(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
281 d = os.path.dirname(fname) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
282 fp = tempfile.NamedTemporaryFile( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
283 prefix='.%s-' % n, suffix='~', dir=d, delete=False |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
284 ) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
285 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
286 yield fp |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
287 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
288 if os.name == 'nt': |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
289 tryunlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
290 os.rename(fp.name, fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
291 finally: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
292 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
293 tryunlink(fp.name) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
294 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
295 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
296 def main(): |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
297 ap = argparse.ArgumentParser() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
298 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
299 '--version', action='version', version='Byteify strings 1.0' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
300 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
301 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
302 '-i', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
303 '--inplace', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
304 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
305 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
306 help='edit files in place', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
307 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
308 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
309 '--dictiter', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
310 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
311 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
312 help='rewrite iteritems() and itervalues()', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
313 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
314 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
315 '--allow-attr-methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
316 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
317 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
318 help='also handle attr*() when they are methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
319 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
320 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
321 '--treat-as-kwargs', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
322 nargs="+", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
323 default=[], |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
324 help="ignore kwargs-like objects", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
325 ), |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
326 ap.add_argument('files', metavar='FILE', nargs='+', help='source file') |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
327 args = ap.parse_args() |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
328 opts = { |
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
329 'dictiter': args.dictiter, |
42679
5e296f618920
byteify-strings: simplify default value for `--treat-as-kwargs`
Raphaël Gomès <rgomes@octobus.net>
parents:
42678
diff
changeset
|
330 'treat-as-kwargs': set(args.treat_as_kwargs), |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
331 'allow-attr-methods': args.allow_attr_methods, |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
332 } |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
333 for fname in args.files: |
44559
bc9a9016467d
byteify-string: resolve symlink before byteifying
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
43407
diff
changeset
|
334 fname = os.path.realpath(fname) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
335 if args.inplace: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
336 with editinplace(fname) as fout: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
337 with open(fname, 'rb') as fin: |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
338 process(fin, fout, opts) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
339 else: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
340 with open(fname, 'rb') as fin: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
341 fout = sys.stdout.buffer |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
342 process(fin, fout, opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
343 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
344 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
345 if __name__ == '__main__': |
43407
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
346 if sys.version_info[0:2] < (3, 7): |
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
347 print('This script must be run under Python 3.7+') |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
348 sys.exit(3) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
349 main() |