Mercurial > hg
annotate contrib/byteify-strings.py @ 46444:6c778d20c8c2
rust: replace ToString impls with Display
ToString is automatically implementing for everything that implements
Display, and Display can avoid allocating intermediate strings.
Differential Revision: https://phab.mercurial-scm.org/D9904
author | Simon Sapin <simon.sapin@octobus.net> |
---|---|
date | Thu, 28 Jan 2021 19:21:57 +0100 |
parents | 89a2afe31e82 |
children | 6000f5b25c9b |
rev | line source |
---|---|
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
1 #!/usr/bin/env python3 |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
2 # |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
3 # byteify-strings.py - transform string literals to be Python 3 safe |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
10 from __future__ import absolute_import, print_function |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
11 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
12 import argparse |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
13 import contextlib |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
14 import errno |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
15 import os |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
16 import sys |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
17 import tempfile |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
18 import token |
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
19 import tokenize |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
21 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
22 def adjusttokenpos(t, ofs): |
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
23 """Adjust start/end column of the given token""" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
24 return t._replace( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
25 start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
26 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
27 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
28 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
29 def replacetokens(tokens, opts): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
30 """Transform a stream of tokens from raw to Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
31 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
32 Returns a generator of possibly rewritten tokens. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
33 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
34 The input token list may be mutated as part of processing. However, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
35 its changes do not necessarily match the output token stream. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
36 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
37 sysstrtokens = set() |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
38 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
39 # The following utility functions access the tokens list and i index of |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
40 # the for i, t enumerate(tokens) loop below |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
41 def _isop(j, *o): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
42 """Assert that tokens[j] is an OP with one of the given values""" |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
43 try: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
44 return tokens[j].type == token.OP and tokens[j].string in o |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
45 except IndexError: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
46 return False |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
47 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
48 def _findargnofcall(n): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
49 """Find arg n of a call expression (start at 0) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
50 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
51 Returns index of the first token of that argument, or None if |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
52 there is not that many arguments. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
53 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
54 Assumes that token[i + 1] is '('. |
38389
1d68fd5f614a
byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents:
38388
diff
changeset
|
55 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
56 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
57 nested = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
58 for j in range(i + 2, len(tokens)): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
59 if _isop(j, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
60 # end of call, tuple, subscription or dict / set |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
61 nested -= 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
62 if nested < 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
63 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
64 elif n == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
65 # this is the starting position of arg |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
66 return j |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
67 elif _isop(j, '(', '[', '{'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
68 nested += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
69 elif _isop(j, ',') and nested == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
70 n -= 1 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
71 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
72 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
73 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
74 def _ensuresysstr(j): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
75 """Make sure the token at j is a system string |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
76 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
77 Remember the given token so the string transformer won't add |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
78 the byte prefix. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
79 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
80 Ignores tokens that are not strings. Assumes bounds checking has |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
81 already been done. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
82 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
83 """ |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
84 k = j |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
85 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
86 while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
87 k += 1 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
88 if currtoken.type == token.STRING and currtoken.string.startswith( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
89 ("'", '"') |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
90 ): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
91 sysstrtokens.add(currtoken) |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
92 try: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
93 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
94 except IndexError: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
95 break |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
96 |
42677
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
97 def _isitemaccess(j): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
98 """Assert the next tokens form an item access on `tokens[j]` and that |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
99 `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
100 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
101 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
102 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
103 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
104 and _isop(j + 1, '[') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
105 and tokens[j + 2].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
106 and _isop(j + 3, ']') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
107 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
108 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
109 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
110 |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
111 def _ismethodcall(j, *methodnames): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
112 """Assert the next tokens form a call to `methodname` with a string |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
113 as first argument on `tokens[j]` and that `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
114 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
115 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
116 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
117 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
118 and _isop(j + 1, '.') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
119 and tokens[j + 2].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
120 and tokens[j + 2].string in methodnames |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
121 and _isop(j + 3, '(') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
122 and tokens[j + 4].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
123 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
124 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
125 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
126 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
127 coldelta = 0 # column increment for new opening parens |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
128 coloffset = -1 # column offset for the current line (-1: TBD) |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
129 parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
130 ignorenextline = False # don't transform the next line |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
131 insideignoreblock = False # don't transform until turned off |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
132 for i, t in enumerate(tokens): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
133 # Compute the column offset for the current line, such that |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
134 # the current line will be aligned to the last opening paren |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
135 # as before. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
136 if coloffset < 0: |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
137 lastparen = parens[-1] |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
138 if t.start[1] == lastparen[1]: |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
139 coloffset = lastparen[2] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
140 elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
141 token.NEWLINE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
142 tokenize.NL, |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
143 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
144 # fix misaligned indent of s/util.Abort/error.Abort/ |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
145 coloffset = lastparen[2] + (lastparen[1] - t.start[1]) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
146 else: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
147 coloffset = 0 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
148 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
149 # Reset per-line attributes at EOL. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
150 if t.type in (token.NEWLINE, tokenize.NL): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
151 yield adjusttokenpos(t, coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
152 coldelta = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
153 coloffset = -1 |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
154 if not insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
155 ignorenextline = ( |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
156 tokens[i - 1].type == token.COMMENT |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
157 and tokens[i - 1].string == "# no-py3-transform" |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
158 ) |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
159 continue |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
160 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
161 if t.type == token.COMMENT: |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
162 if t.string == "# py3-transform: off": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
163 insideignoreblock = True |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
164 if t.string == "# py3-transform: on": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
165 insideignoreblock = False |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
166 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
167 if ignorenextline or insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
168 yield adjusttokenpos(t, coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
169 continue |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
170 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
171 # Remember the last paren position. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
172 if _isop(i, '(', '[', '{'): |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
173 parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type)) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
174 elif _isop(i, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
175 parens.pop() |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
176 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
177 # Convert most string literals to byte literals. String literals |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
178 # in Python 2 are bytes. String literals in Python 3 are unicode. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
179 # Most strings in Mercurial are bytes and unicode strings are rare. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
180 # Rather than rewrite all string literals to use ``b''`` to indicate |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
181 # byte strings, we apply this token transformer to insert the ``b`` |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
182 # prefix nearly everywhere. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
183 if t.type == token.STRING and t not in sysstrtokens: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
184 s = t.string |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
185 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
186 # Preserve docstrings as string literals. This is inconsistent |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
187 # with regular unprefixed strings. However, the |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
188 # "from __future__" parsing (which allows a module docstring to |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
189 # exist before it) doesn't properly handle the docstring if it |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
190 # is b''' prefixed, leading to a SyntaxError. We leave all |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
191 # docstrings as unprefixed to avoid this. This means Mercurial |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
192 # components touching docstrings need to handle unicode, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
193 # unfortunately. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
194 if s[0:3] in ("'''", '"""'): |
42675
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
195 # If it's assigned to something, it's not a docstring |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
196 if not _isop(i - 1, '='): |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
197 yield adjusttokenpos(t, coloffset) |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
198 continue |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
199 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
200 # If the first character isn't a quote, it is likely a string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
201 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
202 if s[0] not in ("'", '"'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
203 yield adjusttokenpos(t, coloffset) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
205 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
206 # String literal. Prefix to make a b'' string. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
207 yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
208 coldelta += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
209 continue |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
210 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
211 # This looks like a function call. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
212 if t.type == token.NAME and _isop(i + 1, '('): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
213 fn = t.string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
214 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
215 # *attr() builtins don't accept byte strings to 2nd argument. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
216 if ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
217 fn |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
218 in ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
219 'getattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
220 'setattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
221 'hasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
222 'safehasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
223 'wrapfunction', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
224 'wrapclass', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
225 'addattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
226 ) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
227 and (opts['allow-attr-methods'] or not _isop(i - 1, '.')) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
228 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
229 arg1idx = _findargnofcall(1) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
230 if arg1idx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
231 _ensuresysstr(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
232 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
233 # .encode() and .decode() on str/bytes/unicode don't accept |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
234 # byte strings on Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
235 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
236 for argn in range(2): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
237 argidx = _findargnofcall(argn) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
238 if argidx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
239 _ensuresysstr(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
240 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
241 # It changes iteritems/values to items/values as they are not |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
242 # present in Python 3 world. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
243 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
244 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
245 continue |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
246 |
42678
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
247 if t.type == token.NAME and t.string in opts['treat-as-kwargs']: |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
248 if _isitemaccess(i): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
249 _ensuresysstr(i + 2) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
250 if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
251 _ensuresysstr(i + 4) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
252 |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
253 # Looks like "if __name__ == '__main__'". |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
254 if ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
255 t.type == token.NAME |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
256 and t.string == '__name__' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
257 and _isop(i + 1, '==') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
258 ): |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
259 _ensuresysstr(i + 2) |
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
260 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
261 # Emit unmodified token. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
262 yield adjusttokenpos(t, coloffset) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
263 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
264 |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
265 def process(fin, fout, opts): |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
266 tokens = tokenize.tokenize(fin.readline) |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
267 tokens = replacetokens(list(tokens), opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
268 fout.write(tokenize.untokenize(tokens)) |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
269 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
270 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
271 def tryunlink(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
272 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
273 os.unlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
274 except OSError as err: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
275 if err.errno != errno.ENOENT: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
276 raise |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
277 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
278 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
279 @contextlib.contextmanager |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
280 def editinplace(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
281 n = os.path.basename(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
282 d = os.path.dirname(fname) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
283 fp = tempfile.NamedTemporaryFile( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
284 prefix='.%s-' % n, suffix='~', dir=d, delete=False |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
285 ) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
286 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
287 yield fp |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
288 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
289 if os.name == 'nt': |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
290 tryunlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
291 os.rename(fp.name, fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
292 finally: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
293 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
294 tryunlink(fp.name) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
295 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
296 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
297 def main(): |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
298 ap = argparse.ArgumentParser() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
299 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
300 '--version', action='version', version='Byteify strings 1.0' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
301 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
302 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
303 '-i', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
304 '--inplace', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
305 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
306 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
307 help='edit files in place', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
308 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
309 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
310 '--dictiter', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
311 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
312 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
313 help='rewrite iteritems() and itervalues()', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
314 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
315 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
316 '--allow-attr-methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
317 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
318 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
319 help='also handle attr*() when they are methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
320 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
321 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
322 '--treat-as-kwargs', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
323 nargs="+", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
324 default=[], |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
325 help="ignore kwargs-like objects", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
326 ), |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
327 ap.add_argument('files', metavar='FILE', nargs='+', help='source file') |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
328 args = ap.parse_args() |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
329 opts = { |
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
330 'dictiter': args.dictiter, |
42679
5e296f618920
byteify-strings: simplify default value for `--treat-as-kwargs`
Raphaël Gomès <rgomes@octobus.net>
parents:
42678
diff
changeset
|
331 'treat-as-kwargs': set(args.treat_as_kwargs), |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
332 'allow-attr-methods': args.allow_attr_methods, |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
333 } |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
334 for fname in args.files: |
44559
bc9a9016467d
byteify-string: resolve symlink before byteifying
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
43407
diff
changeset
|
335 fname = os.path.realpath(fname) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
336 if args.inplace: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
337 with editinplace(fname) as fout: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
338 with open(fname, 'rb') as fin: |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
339 process(fin, fout, opts) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
340 else: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
341 with open(fname, 'rb') as fin: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
342 fout = sys.stdout.buffer |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
343 process(fin, fout, opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
344 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
345 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
346 if __name__ == '__main__': |
43407
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
347 if sys.version_info[0:2] < (3, 7): |
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
348 print('This script must be run under Python 3.7+') |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
349 sys.exit(3) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
350 main() |