patch: do not break up multibyte character when highlighting word
This changes {\W} to {\W - any 8bit characters} so that multibyte sequences
are taken as words. Since we don't know the encoding of user content, this
is the most sensible definition of a non-word.
--- a/mercurial/patch.py Sun Dec 10 00:16:11 2017 -0500
+++ b/mercurial/patch.py Mon Dec 11 22:38:31 2017 +0900
@@ -46,6 +46,7 @@
gitre = re.compile(br'diff --git a/(.*) b/(.*)')
tabsplitter = re.compile(br'(\t+|[^\t]+)')
+_nonwordre = re.compile(br'([^a-zA-Z0-9_\x80-\xff])')
PatchError = error.PatchError
@@ -2578,7 +2579,7 @@
raise error.ProgrammingError("Case not expected, operation = %s" %
operation)
- s = difflib.ndiff(re.split(br'(\W)', s2), re.split(br'(\W)', s1))
+ s = difflib.ndiff(_nonwordre.split(s2), _nonwordre.split(s1))
for part in s:
if part[0] in operation_skip or len(part) == 2:
continue
--- a/tests/test-diff-color.t Sun Dec 10 00:16:11 2017 -0500
+++ b/tests/test-diff-color.t Mon Dec 11 22:38:31 2017 +0900
@@ -370,3 +370,23 @@
[diff.deleted|-(to see if it works)]
[diff.inserted|+three of those lines ][diff.inserted.highlight|have]
[diff.inserted|+][diff.inserted.highlight|collapsed][diff.inserted| onto one]
+
+multibyte character shouldn't be broken up in word diff:
+
+ $ $PYTHON <<'EOF'
+ > with open("utf8", "wb") as f:
+ > f.write(b"blah \xe3\x82\xa2 blah\n")
+ > EOF
+ $ hg ci -Am 'add utf8 char' utf8
+ $ $PYTHON <<'EOF'
+ > with open("utf8", "wb") as f:
+ > f.write(b"blah \xe3\x82\xa4 blah\n")
+ > EOF
+ $ hg ci -m 'slightly change utf8 char' utf8
+ $ hg diff --config experimental.worddiff=True --color=debug -c.
+ [diff.diffline|diff --git a/utf8 b/utf8]
+ [diff.file_a|--- a/utf8]
+ [diff.file_b|+++ b/utf8]
+ [diff.hunk|@@ -1,1 +1,1 @@]
+ [diff.deleted|-blah ][diff.deleted.highlight|\xe3\x82\xa2][diff.deleted| blah] (esc)
+ [diff.inserted|+blah ][diff.inserted.highlight|\xe3\x82\xa4][diff.inserted| blah] (esc)