Mercurial > hg
annotate contrib/byteify-strings.py @ 50637:32837c7e2e4b
revlog: add a `get_revlog` method
This might seen weird, but I actually thing we have been needing this for a
long time. There is multiple object that kind of pretend being revlogs while
actually wrapping the actual revlog. Since multiple code needs to access the
actuel revlog. See documentation for more details.
Expect cleanup of various places one the current series is done.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Sun, 28 May 2023 05:23:46 +0200 |
parents | 6000f5b25c9b |
children | 8250ecb53f30 |
rev | line source |
---|---|
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
1 #!/usr/bin/env python3 |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
2 # |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
3 # byteify-strings.py - transform string literals to be Python 3 safe |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
10 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
11 import argparse |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
12 import contextlib |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
13 import errno |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
14 import os |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
15 import sys |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
16 import tempfile |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
17 import token |
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
18 import tokenize |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
19 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
20 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
21 def adjusttokenpos(t, ofs): |
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
22 """Adjust start/end column of the given token""" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
23 return t._replace( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
24 start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
25 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
26 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
27 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
28 def replacetokens(tokens, opts): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
29 """Transform a stream of tokens from raw to Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
30 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
31 Returns a generator of possibly rewritten tokens. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
32 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
33 The input token list may be mutated as part of processing. However, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
34 its changes do not necessarily match the output token stream. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
35 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
36 sysstrtokens = set() |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
37 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
38 # The following utility functions access the tokens list and i index of |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
39 # the for i, t enumerate(tokens) loop below |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
40 def _isop(j, *o): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
41 """Assert that tokens[j] is an OP with one of the given values""" |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
42 try: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
43 return tokens[j].type == token.OP and tokens[j].string in o |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
44 except IndexError: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
45 return False |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
46 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
47 def _findargnofcall(n): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
48 """Find arg n of a call expression (start at 0) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
49 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
50 Returns index of the first token of that argument, or None if |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
51 there is not that many arguments. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
52 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
53 Assumes that token[i + 1] is '('. |
38389
1d68fd5f614a
byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents:
38388
diff
changeset
|
54 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
55 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
56 nested = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
57 for j in range(i + 2, len(tokens)): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
58 if _isop(j, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
59 # end of call, tuple, subscription or dict / set |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
60 nested -= 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
61 if nested < 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
62 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
63 elif n == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
64 # this is the starting position of arg |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
65 return j |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
66 elif _isop(j, '(', '[', '{'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
67 nested += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
68 elif _isop(j, ',') and nested == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
69 n -= 1 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
70 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
71 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
72 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
73 def _ensuresysstr(j): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
74 """Make sure the token at j is a system string |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
75 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
76 Remember the given token so the string transformer won't add |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
77 the byte prefix. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
78 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
79 Ignores tokens that are not strings. Assumes bounds checking has |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
80 already been done. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
81 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
82 """ |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
83 k = j |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
84 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
85 while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
86 k += 1 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
87 if currtoken.type == token.STRING and currtoken.string.startswith( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
88 ("'", '"') |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
89 ): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
90 sysstrtokens.add(currtoken) |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
91 try: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
92 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
93 except IndexError: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
94 break |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
95 |
42677
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
96 def _isitemaccess(j): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
97 """Assert the next tokens form an item access on `tokens[j]` and that |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
98 `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
99 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
100 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
101 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
102 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
103 and _isop(j + 1, '[') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
104 and tokens[j + 2].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
105 and _isop(j + 3, ']') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
106 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
107 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
108 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
109 |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
110 def _ismethodcall(j, *methodnames): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
111 """Assert the next tokens form a call to `methodname` with a string |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
112 as first argument on `tokens[j]` and that `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
113 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
114 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
115 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
116 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
117 and _isop(j + 1, '.') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
118 and tokens[j + 2].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
119 and tokens[j + 2].string in methodnames |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
120 and _isop(j + 3, '(') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
121 and tokens[j + 4].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
122 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
123 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
124 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
125 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
126 coldelta = 0 # column increment for new opening parens |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
127 coloffset = -1 # column offset for the current line (-1: TBD) |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
128 parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
129 ignorenextline = False # don't transform the next line |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
130 insideignoreblock = False # don't transform until turned off |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
131 for i, t in enumerate(tokens): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
132 # Compute the column offset for the current line, such that |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
133 # the current line will be aligned to the last opening paren |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
134 # as before. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
135 if coloffset < 0: |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
136 lastparen = parens[-1] |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
137 if t.start[1] == lastparen[1]: |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
138 coloffset = lastparen[2] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
139 elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
140 token.NEWLINE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
141 tokenize.NL, |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
142 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
143 # fix misaligned indent of s/util.Abort/error.Abort/ |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
144 coloffset = lastparen[2] + (lastparen[1] - t.start[1]) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
145 else: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
146 coloffset = 0 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
147 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
148 # Reset per-line attributes at EOL. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
149 if t.type in (token.NEWLINE, tokenize.NL): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
150 yield adjusttokenpos(t, coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
151 coldelta = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
152 coloffset = -1 |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
153 if not insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
154 ignorenextline = ( |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
155 tokens[i - 1].type == token.COMMENT |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
156 and tokens[i - 1].string == "# no-py3-transform" |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
157 ) |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
158 continue |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
159 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
160 if t.type == token.COMMENT: |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
161 if t.string == "# py3-transform: off": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
162 insideignoreblock = True |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
163 if t.string == "# py3-transform: on": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
164 insideignoreblock = False |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
165 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
166 if ignorenextline or insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
167 yield adjusttokenpos(t, coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
168 continue |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
169 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
170 # Remember the last paren position. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
171 if _isop(i, '(', '[', '{'): |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
172 parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type)) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
173 elif _isop(i, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
174 parens.pop() |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
175 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
176 # Convert most string literals to byte literals. String literals |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
177 # in Python 2 are bytes. String literals in Python 3 are unicode. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
178 # Most strings in Mercurial are bytes and unicode strings are rare. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
179 # Rather than rewrite all string literals to use ``b''`` to indicate |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
180 # byte strings, we apply this token transformer to insert the ``b`` |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
181 # prefix nearly everywhere. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
182 if t.type == token.STRING and t not in sysstrtokens: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
183 s = t.string |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
184 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
185 # Preserve docstrings as string literals. This is inconsistent |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
186 # with regular unprefixed strings. However, the |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
187 # "from __future__" parsing (which allows a module docstring to |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
188 # exist before it) doesn't properly handle the docstring if it |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
189 # is b''' prefixed, leading to a SyntaxError. We leave all |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
190 # docstrings as unprefixed to avoid this. This means Mercurial |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
191 # components touching docstrings need to handle unicode, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
192 # unfortunately. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
193 if s[0:3] in ("'''", '"""'): |
42675
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
194 # If it's assigned to something, it's not a docstring |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
195 if not _isop(i - 1, '='): |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
196 yield adjusttokenpos(t, coloffset) |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
197 continue |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
198 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
199 # If the first character isn't a quote, it is likely a string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
200 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
201 if s[0] not in ("'", '"'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
202 yield adjusttokenpos(t, coloffset) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
203 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
205 # String literal. Prefix to make a b'' string. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
206 yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
207 coldelta += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
208 continue |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
209 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
210 # This looks like a function call. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
211 if t.type == token.NAME and _isop(i + 1, '('): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
212 fn = t.string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
213 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
214 # *attr() builtins don't accept byte strings to 2nd argument. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
215 if ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
216 fn |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
217 in ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
218 'getattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
219 'setattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
220 'hasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
221 'safehasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
222 'wrapfunction', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
223 'wrapclass', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
224 'addattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
225 ) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
226 and (opts['allow-attr-methods'] or not _isop(i - 1, '.')) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
227 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
228 arg1idx = _findargnofcall(1) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
229 if arg1idx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
230 _ensuresysstr(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
231 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
232 # .encode() and .decode() on str/bytes/unicode don't accept |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
233 # byte strings on Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
234 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
235 for argn in range(2): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
236 argidx = _findargnofcall(argn) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
237 if argidx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
238 _ensuresysstr(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
239 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
240 # It changes iteritems/values to items/values as they are not |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
241 # present in Python 3 world. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
242 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
243 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
244 continue |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
245 |
42678
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
246 if t.type == token.NAME and t.string in opts['treat-as-kwargs']: |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
247 if _isitemaccess(i): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
248 _ensuresysstr(i + 2) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
249 if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
250 _ensuresysstr(i + 4) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
251 |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
252 # Looks like "if __name__ == '__main__'". |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
253 if ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
254 t.type == token.NAME |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
255 and t.string == '__name__' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
256 and _isop(i + 1, '==') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
257 ): |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
258 _ensuresysstr(i + 2) |
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
259 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
260 # Emit unmodified token. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
261 yield adjusttokenpos(t, coloffset) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
262 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
263 |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
264 def process(fin, fout, opts): |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
265 tokens = tokenize.tokenize(fin.readline) |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
266 tokens = replacetokens(list(tokens), opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
267 fout.write(tokenize.untokenize(tokens)) |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
268 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
269 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
270 def tryunlink(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
271 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
272 os.unlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
273 except OSError as err: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
274 if err.errno != errno.ENOENT: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
275 raise |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
276 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
277 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
278 @contextlib.contextmanager |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
279 def editinplace(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
280 n = os.path.basename(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
281 d = os.path.dirname(fname) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
282 fp = tempfile.NamedTemporaryFile( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
283 prefix='.%s-' % n, suffix='~', dir=d, delete=False |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
284 ) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
285 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
286 yield fp |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
287 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
288 if os.name == 'nt': |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
289 tryunlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
290 os.rename(fp.name, fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
291 finally: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
292 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
293 tryunlink(fp.name) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
294 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
295 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
296 def main(): |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
297 ap = argparse.ArgumentParser() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
298 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
299 '--version', action='version', version='Byteify strings 1.0' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
300 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
301 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
302 '-i', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
303 '--inplace', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
304 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
305 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
306 help='edit files in place', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
307 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
308 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
309 '--dictiter', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
310 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
311 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
312 help='rewrite iteritems() and itervalues()', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
313 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
314 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
315 '--allow-attr-methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
316 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
317 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
318 help='also handle attr*() when they are methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
319 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
320 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
321 '--treat-as-kwargs', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
322 nargs="+", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
323 default=[], |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
324 help="ignore kwargs-like objects", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
325 ), |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
326 ap.add_argument('files', metavar='FILE', nargs='+', help='source file') |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
327 args = ap.parse_args() |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
328 opts = { |
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
329 'dictiter': args.dictiter, |
42679
5e296f618920
byteify-strings: simplify default value for `--treat-as-kwargs`
Raphaël Gomès <rgomes@octobus.net>
parents:
42678
diff
changeset
|
330 'treat-as-kwargs': set(args.treat_as_kwargs), |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
331 'allow-attr-methods': args.allow_attr_methods, |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
332 } |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
333 for fname in args.files: |
44559
bc9a9016467d
byteify-string: resolve symlink before byteifying
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
43407
diff
changeset
|
334 fname = os.path.realpath(fname) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
335 if args.inplace: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
336 with editinplace(fname) as fout: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
337 with open(fname, 'rb') as fin: |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
338 process(fin, fout, opts) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
339 else: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
340 with open(fname, 'rb') as fin: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
341 fout = sys.stdout.buffer |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
342 process(fin, fout, opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
343 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
344 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
345 if __name__ == '__main__': |
43407
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
346 if sys.version_info[0:2] < (3, 7): |
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
347 print('This script must be run under Python 3.7+') |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
348 sys.exit(3) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
349 main() |