run-tests: explicitly handle unicode when writing xunit file

run-tests: explicitly handle unicode when writing xunit file

The xunit writer was passing a str to a minidom API. An implicit
.decode('ascii') was performed somewhere, causing UnicodeDecodeError
if test output contained non-ascii sequences.

This patch converts test output to utf-8 before passing it to minidom.
We use the "replace" strategy to ensure invalid utf-8 sequences get
munged into �.

--- a/tests/run-tests.py	Sun Mar 29 19:06:23 2015 +0200
+++ b/tests/run-tests.py	Sun Mar 29 10:41:23 2015 -0700
@@ -1469,7 +1469,11 @@
                     t = doc.createElement('testcase')
                     t.setAttribute('name', tc)
                     t.setAttribute('time', '%.3f' % timesd[tc])
-                    cd = doc.createCDATASection(cdatasafe(err))
+                    # createCDATASection expects a unicode or it will convert
+                    # using default conversion rules, which will fail if
+                    # string isn't ASCII.
+                    err = cdatasafe(err).decode('utf-8', 'replace')
+                    cd = doc.createCDATASection(err)
                     t.appendChild(cd)
                     s.appendChild(t)
                 xuf.write(doc.toprettyxml(indent='  ', encoding='utf-8'))

--- a/tests/test-run-tests.t	Sun Mar 29 19:06:23 2015 +0200
+++ b/tests/test-run-tests.t	Sun Mar 29 10:41:23 2015 -0700
@@ -31,6 +31,10 @@
   > this test is still more bytes than success.
   > EOF
 
+  >>> fh = open('test-failure-unicode.t', 'wb')
+  >>> fh.write(u'  $ echo babar\u03b1\n'.encode('utf-8'))
+  >>> fh.write(u'  l\u03b5\u03b5t\n'.encode('utf-8'))
+
   $ $TESTDIR/run-tests.py --with-hg=`which hg`
   
   --- $TESTTMP/test-failure.t
@@ -44,10 +48,21 @@
   
   ERROR: test-failure.t output changed
   !.
+  --- $TESTTMP/test-failure-unicode.t
+  +++ $TESTTMP/test-failure-unicode.t.err
+  @@ -1,2 +1,2 @@
+     $ echo babar\xce\xb1 (esc)
+  -  l\xce\xb5\xce\xb5t (esc)
+  +  babar\xce\xb1 (esc)
+  
+  ERROR: test-failure-unicode.t output changed
+  !
   Failed test-failure.t: output changed
-  # Ran 2 tests, 0 skipped, 0 warned, 1 failed.
+  Failed test-failure-unicode.t: output changed
+  # Ran 3 tests, 0 skipped, 0 warned, 2 failed.
   python hash seed: * (glob)
   [1]
+
 test --xunit support
   $ $TESTDIR/run-tests.py --with-hg=`which hg` --xunit=xunit.xml
   
@@ -62,14 +77,32 @@
   
   ERROR: test-failure.t output changed
   !.
+  --- $TESTTMP/test-failure-unicode.t
+  +++ $TESTTMP/test-failure-unicode.t.err
+  @@ -1,2 +1,2 @@
+     $ echo babar\xce\xb1 (esc)
+  -  l\xce\xb5\xce\xb5t (esc)
+  +  babar\xce\xb1 (esc)
+  
+  ERROR: test-failure-unicode.t output changed
+  !
   Failed test-failure.t: output changed
-  # Ran 2 tests, 0 skipped, 0 warned, 1 failed.
+  Failed test-failure-unicode.t: output changed
+  # Ran 3 tests, 0 skipped, 0 warned, 2 failed.
   python hash seed: * (glob)
   [1]
   $ cat xunit.xml
   <?xml version="1.0" encoding="utf-8"?>
-  <testsuite errors="0" failures="1" name="run-tests" skipped="0" tests="2">
+  <testsuite errors="0" failures="2" name="run-tests" skipped="0" tests="3">
     <testcase name="test-success.t" time="*"/> (glob)
+    <testcase name="test-failure-unicode.t" time="*"> (glob)
+  <![CDATA[--- $TESTTMP/test-failure-unicode.t
+  +++ $TESTTMP/test-failure-unicode.t.err
+  @@ -1,2 +1,2 @@
+     $ echo babar\xce\xb1 (esc)
+  -  l\xce\xb5\xce\xb5t (esc)
+  +  babar\xce\xb1 (esc)
+  ]]>  </testcase>
     <testcase name="test-failure.t" time="*"> (glob)
   <![CDATA[--- $TESTTMP/test-failure.t
   +++ $TESTTMP/test-failure.t.err
@@ -82,6 +115,8 @@
   ]]>  </testcase>
   </testsuite>
 
+  $ rm test-failure-unicode.t
+
 test for --retest
 ====================