Mercurial > hg
annotate contrib/byteify-strings.py @ 46370:cad17d50736c
changelog: move branchinfo to changelogrevision
The function parses the extra dictionary after looking up the
changelogrevision. To avoid duplicated look up, it is better to provide
it as property of changelogrevision instead. Keep the function for a
release cycle as at least the topic extension depends on it.
Differential Revision: https://phab.mercurial-scm.org/D9779
author | Joerg Sonnenberger <joerg@bec.de> |
---|---|
date | Fri, 15 Jan 2021 01:30:08 +0100 |
parents | 89a2afe31e82 |
children | 6000f5b25c9b |
rev | line source |
---|---|
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
1 #!/usr/bin/env python3 |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
2 # |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
3 # byteify-strings.py - transform string literals to be Python 3 safe |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
4 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
5 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
6 # |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
7 # This software may be used and distributed according to the terms of the |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
8 # GNU General Public License version 2 or any later version. |
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
9 |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
10 from __future__ import absolute_import, print_function |
27220
4374d819ccd5
mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents:
0
diff
changeset
|
11 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
12 import argparse |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
13 import contextlib |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
14 import errno |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
15 import os |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
16 import sys |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
17 import tempfile |
38384
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
18 import token |
1d9c97db465f
byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents:
36617
diff
changeset
|
19 import tokenize |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
20 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
21 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
22 def adjusttokenpos(t, ofs): |
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
23 """Adjust start/end column of the given token""" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
24 return t._replace( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
25 start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
26 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
27 |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
28 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
29 def replacetokens(tokens, opts): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
30 """Transform a stream of tokens from raw to Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
31 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
32 Returns a generator of possibly rewritten tokens. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
33 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
34 The input token list may be mutated as part of processing. However, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
35 its changes do not necessarily match the output token stream. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
36 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
37 sysstrtokens = set() |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
38 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
39 # The following utility functions access the tokens list and i index of |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
40 # the for i, t enumerate(tokens) loop below |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
41 def _isop(j, *o): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
42 """Assert that tokens[j] is an OP with one of the given values""" |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
43 try: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
44 return tokens[j].type == token.OP and tokens[j].string in o |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
45 except IndexError: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
46 return False |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
47 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
48 def _findargnofcall(n): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
49 """Find arg n of a call expression (start at 0) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
50 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
51 Returns index of the first token of that argument, or None if |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
52 there is not that many arguments. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
53 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
54 Assumes that token[i + 1] is '('. |
38389
1d68fd5f614a
byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents:
38388
diff
changeset
|
55 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
56 """ |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
57 nested = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
58 for j in range(i + 2, len(tokens)): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
59 if _isop(j, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
60 # end of call, tuple, subscription or dict / set |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
61 nested -= 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
62 if nested < 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
63 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
64 elif n == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
65 # this is the starting position of arg |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
66 return j |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
67 elif _isop(j, '(', '[', '{'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
68 nested += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
69 elif _isop(j, ',') and nested == 0: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
70 n -= 1 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
71 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
72 return None |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
73 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
74 def _ensuresysstr(j): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
75 """Make sure the token at j is a system string |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
76 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
77 Remember the given token so the string transformer won't add |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
78 the byte prefix. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
79 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
80 Ignores tokens that are not strings. Assumes bounds checking has |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
81 already been done. |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
82 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
83 """ |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
84 k = j |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
85 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
86 while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
87 k += 1 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
88 if currtoken.type == token.STRING and currtoken.string.startswith( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
89 ("'", '"') |
42674
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
90 ): |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
91 sysstrtokens.add(currtoken) |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
92 try: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
93 currtoken = tokens[k] |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
94 except IndexError: |
70bd1965bd07
byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents:
42247
diff
changeset
|
95 break |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
96 |
42677
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
97 def _isitemaccess(j): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
98 """Assert the next tokens form an item access on `tokens[j]` and that |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
99 `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
100 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
101 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
102 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
103 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
104 and _isop(j + 1, '[') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
105 and tokens[j + 2].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
106 and _isop(j + 3, ']') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
107 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
108 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
109 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
110 |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
111 def _ismethodcall(j, *methodnames): |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
112 """Assert the next tokens form a call to `methodname` with a string |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
113 as first argument on `tokens[j]` and that `tokens[j]` is a name. |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
114 """ |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
115 try: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
116 return ( |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
117 tokens[j].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
118 and _isop(j + 1, '.') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
119 and tokens[j + 2].type == token.NAME |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
120 and tokens[j + 2].string in methodnames |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
121 and _isop(j + 3, '(') |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
122 and tokens[j + 4].type == token.STRING |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
123 ) |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
124 except IndexError: |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
125 return False |
c9fd8163131f
byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents:
42676
diff
changeset
|
126 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
127 coldelta = 0 # column increment for new opening parens |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
128 coloffset = -1 # column offset for the current line (-1: TBD) |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
129 parens = [(0, 0, 0, -1)] # stack of (line, end-column, column-offset, type) |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
130 ignorenextline = False # don't transform the next line |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
131 insideignoreblock = False # don't transform until turned off |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
132 for i, t in enumerate(tokens): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
133 # Compute the column offset for the current line, such that |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
134 # the current line will be aligned to the last opening paren |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
135 # as before. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
136 if coloffset < 0: |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
137 lastparen = parens[-1] |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
138 if t.start[1] == lastparen[1]: |
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
139 coloffset = lastparen[2] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
140 elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
141 token.NEWLINE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
142 tokenize.NL, |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
143 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
144 # fix misaligned indent of s/util.Abort/error.Abort/ |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
145 coloffset = lastparen[2] + (lastparen[1] - t.start[1]) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
146 else: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
147 coloffset = 0 |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
148 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
149 # Reset per-line attributes at EOL. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
150 if t.type in (token.NEWLINE, tokenize.NL): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
151 yield adjusttokenpos(t, coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
152 coldelta = 0 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
153 coloffset = -1 |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
154 if not insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
155 ignorenextline = ( |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
156 tokens[i - 1].type == token.COMMENT |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
157 and tokens[i - 1].string == "# no-py3-transform" |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
158 ) |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
159 continue |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
160 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
161 if t.type == token.COMMENT: |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
162 if t.string == "# py3-transform: off": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
163 insideignoreblock = True |
42700
f9b64ff9d26b
byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents:
42687
diff
changeset
|
164 if t.string == "# py3-transform: on": |
42676
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
165 insideignoreblock = False |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
166 |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
167 if ignorenextline or insideignoreblock: |
b9a200477edf
byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents:
42675
diff
changeset
|
168 yield adjusttokenpos(t, coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
169 continue |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
170 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
171 # Remember the last paren position. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
172 if _isop(i, '(', '[', '{'): |
42687
26a31c88e1a5
byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents:
42683
diff
changeset
|
173 parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type)) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
174 elif _isop(i, ')', ']', '}'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
175 parens.pop() |
30165
423377290a3a
py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents:
30118
diff
changeset
|
176 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
177 # Convert most string literals to byte literals. String literals |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
178 # in Python 2 are bytes. String literals in Python 3 are unicode. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
179 # Most strings in Mercurial are bytes and unicode strings are rare. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
180 # Rather than rewrite all string literals to use ``b''`` to indicate |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
181 # byte strings, we apply this token transformer to insert the ``b`` |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
182 # prefix nearly everywhere. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
183 if t.type == token.STRING and t not in sysstrtokens: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
184 s = t.string |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
185 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
186 # Preserve docstrings as string literals. This is inconsistent |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
187 # with regular unprefixed strings. However, the |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
188 # "from __future__" parsing (which allows a module docstring to |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
189 # exist before it) doesn't properly handle the docstring if it |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
190 # is b''' prefixed, leading to a SyntaxError. We leave all |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
191 # docstrings as unprefixed to avoid this. This means Mercurial |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
192 # components touching docstrings need to handle unicode, |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
193 # unfortunately. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
194 if s[0:3] in ("'''", '"""'): |
42675
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
195 # If it's assigned to something, it's not a docstring |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
196 if not _isop(i - 1, '='): |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
197 yield adjusttokenpos(t, coloffset) |
e9592e113c31
byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents:
42674
diff
changeset
|
198 continue |
38390
47dd23e6b116
byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents:
38389
diff
changeset
|
199 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
200 # If the first character isn't a quote, it is likely a string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
201 # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
202 if s[0] not in ("'", '"'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
203 yield adjusttokenpos(t, coloffset) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
204 continue |
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
205 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
206 # String literal. Prefix to make a b'' string. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
207 yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset) |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
208 coldelta += 1 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
209 continue |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
210 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
211 # This looks like a function call. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
212 if t.type == token.NAME and _isop(i + 1, '('): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
213 fn = t.string |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
214 |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
215 # *attr() builtins don't accept byte strings to 2nd argument. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
216 if ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
217 fn |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
218 in ( |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
219 'getattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
220 'setattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
221 'hasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
222 'safehasattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
223 'wrapfunction', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
224 'wrapclass', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
225 'addattr', |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
226 ) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
227 and (opts['allow-attr-methods'] or not _isop(i - 1, '.')) |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
44559
diff
changeset
|
228 ): |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
229 arg1idx = _findargnofcall(1) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
230 if arg1idx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
231 _ensuresysstr(arg1idx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
232 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
233 # .encode() and .decode() on str/bytes/unicode don't accept |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
234 # byte strings on Python 3. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
235 elif fn in ('encode', 'decode') and _isop(i - 1, '.'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
236 for argn in range(2): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
237 argidx = _findargnofcall(argn) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
238 if argidx is not None: |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
239 _ensuresysstr(argidx) |
29550
1c22400db72d
mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29490
diff
changeset
|
240 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
241 # It changes iteritems/values to items/values as they are not |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
242 # present in Python 3 world. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
243 elif opts['dictiter'] and fn in ('iteritems', 'itervalues'): |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
244 yield adjusttokenpos(t._replace(string=fn[4:]), coloffset) |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
245 continue |
30052
eaaedad68011
py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30051
diff
changeset
|
246 |
42678
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
247 if t.type == token.NAME and t.string in opts['treat-as-kwargs']: |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
248 if _isitemaccess(i): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
249 _ensuresysstr(i + 2) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
250 if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'): |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
251 _ensuresysstr(i + 4) |
f95b59ffc307
byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents:
42677
diff
changeset
|
252 |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
253 # Looks like "if __name__ == '__main__'". |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
254 if ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
255 t.type == token.NAME |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
256 and t.string == '__name__' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
257 and _isop(i + 1, '==') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
258 ): |
39103
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
259 _ensuresysstr(i + 2) |
da130c5cef90
byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents:
38391
diff
changeset
|
260 |
38391
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
261 # Emit unmodified token. |
f77bbd34a1df
byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents:
38390
diff
changeset
|
262 yield adjusttokenpos(t, coloffset) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
263 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
264 |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
265 def process(fin, fout, opts): |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
266 tokens = tokenize.tokenize(fin.readline) |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
267 tokens = replacetokens(list(tokens), opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
268 fout.write(tokenize.untokenize(tokens)) |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
269 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
270 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
271 def tryunlink(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
272 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
273 os.unlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
274 except OSError as err: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
275 if err.errno != errno.ENOENT: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
276 raise |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
277 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
278 |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
279 @contextlib.contextmanager |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
280 def editinplace(fname): |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
281 n = os.path.basename(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
282 d = os.path.dirname(fname) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
283 fp = tempfile.NamedTemporaryFile( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
284 prefix='.%s-' % n, suffix='~', dir=d, delete=False |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
285 ) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
286 try: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
287 yield fp |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
288 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
289 if os.name == 'nt': |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
290 tryunlink(fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
291 os.rename(fp.name, fname) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
292 finally: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
293 fp.close() |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
294 tryunlink(fp.name) |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
295 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
296 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
297 def main(): |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
298 ap = argparse.ArgumentParser() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
299 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
300 '--version', action='version', version='Byteify strings 1.0' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
301 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
302 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
303 '-i', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
304 '--inplace', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
305 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
306 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
307 help='edit files in place', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
308 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
309 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
310 '--dictiter', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
311 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
312 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
313 help='rewrite iteritems() and itervalues()', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
314 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
315 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
316 '--allow-attr-methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
317 action='store_true', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
318 default=False, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
319 help='also handle attr*() when they are methods', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
320 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
321 ap.add_argument( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
322 '--treat-as-kwargs', |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
323 nargs="+", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
324 default=[], |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
325 help="ignore kwargs-like objects", |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
326 ), |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
327 ap.add_argument('files', metavar='FILE', nargs='+', help='source file') |
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
328 args = ap.parse_args() |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
329 opts = { |
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
330 'dictiter': args.dictiter, |
42679
5e296f618920
byteify-strings: simplify default value for `--treat-as-kwargs`
Raphaël Gomès <rgomes@octobus.net>
parents:
42678
diff
changeset
|
331 'treat-as-kwargs': set(args.treat_as_kwargs), |
42683
bbb002b378f3
byteify-strings: add cli argument to handle `attr*()` when they are methods
Raphaël Gomès <rgomes@octobus.net>
parents:
42679
diff
changeset
|
332 'allow-attr-methods': args.allow_attr_methods, |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
333 } |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
334 for fname in args.files: |
44559
bc9a9016467d
byteify-string: resolve symlink before byteifying
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
43407
diff
changeset
|
335 fname = os.path.realpath(fname) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
336 if args.inplace: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
337 with editinplace(fname) as fout: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
338 with open(fname, 'rb') as fin: |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
339 process(fin, fout, opts) |
38386
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
340 else: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
341 with open(fname, 'rb') as fin: |
9f42e4a83676
byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents:
38385
diff
changeset
|
342 fout = sys.stdout.buffer |
38388
f701bc936e7f
byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents:
38387
diff
changeset
|
343 process(fin, fout, opts) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
344 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42701
diff
changeset
|
345 |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
346 if __name__ == '__main__': |
43407
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
347 if sys.version_info[0:2] < (3, 7): |
09234eeae206
contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents:
43379
diff
changeset
|
348 print('This script must be run under Python 3.7+') |
42247
970aaf38c3fc
contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents:
39103
diff
changeset
|
349 sys.exit(3) |
38385
a2976c27dac4
byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents:
38384
diff
changeset
|
350 main() |