changeset 42676:b9a200477edf

byteify-strings: add support for ignore comments

Our simple token analysis is sometimes not clever enough, we need to be able
to turn off our script for parts of the code.

This change introduces three special comments:

    - `#no-py3-transform` to tell `byteify-strings` ignore the next line
    - `#py3-transform: off` to ignore everything until the end of the file
    - `#py3-transform: on` to stop ignoring

The last two can be particularly useful within Python 2/3 compatibility files.

--- a/contrib/byteify-strings.py	Fri Aug 02 09:48:13 2019 +0200
+++ b/contrib/byteify-strings.py	Fri Aug 02 09:55:32 2019 +0200
@@ -95,6 +95,8 @@
     coldelta = 0  # column increment for new opening parens
     coloffset = -1  # column offset for the current line (-1: TBD)
     parens = [(0, 0, 0)]  # stack of (line, end-column, column-offset)
+    ignorenextline = False  # don't transform the next line
+    insideignoreblock = False # don't transform until turned off
     for i, t in enumerate(tokens):
         # Compute the column offset for the current line, such that
         # the current line will be aligned to the last opening paren
@@ -113,6 +115,21 @@
             yield adjusttokenpos(t, coloffset)
             coldelta = 0
             coloffset = -1
+            if not insideignoreblock:
+                ignorenextline = (
+                    tokens[i - 1].type == token.COMMENT
+                    and tokens[i - 1].string == "#no-py3-transform"
+                )
+            continue
+
+        if t.type == token.COMMENT:
+            if t.string == "#py3-transform: off":
+                insideignoreblock = True
+            if t.string == "#py3-transform: on":
+                insideignoreblock = False
+
+        if ignorenextline or insideignoreblock:
+            yield adjusttokenpos(t, coloffset)
             continue
 
         # Remember the last paren position.