changeset 34047:e97be042fa1b

encoding: check overflow while calculating size of JSON escape buffer

The minimum input size to exploit is ~682MB (= INT_MAX / len('\\u0000') * 2)
on 32bit system, which isn't easy to achieve using Python str in 2GB process
address space, but probably doable.

--- a/mercurial/cext/charencode.c	Wed Aug 30 20:25:56 2017 +0000
+++ b/mercurial/cext/charencode.c	Thu Aug 31 21:56:40 2017 +0900
@@ -294,11 +294,21 @@
 				return -1;
 			}
 			esclen += jsonparanoidlentable[(unsigned char)c];
+			if (esclen < 0) {
+				PyErr_SetString(PyExc_MemoryError,
+						"overflow in jsonescapelen");
+				return -1;
+			}
 		}
 	} else {
 		for (i = 0; i < len; i++) {
 			char c = buf[i];
 			esclen += jsonlentable[(unsigned char)c];
+			if (esclen < 0) {
+				PyErr_SetString(PyExc_MemoryError,
+						"overflow in jsonescapelen");
+				return -1;
+			}
 		}
 	}
 
@@ -366,7 +376,7 @@
 	origlen = PyBytes_GET_SIZE(origstr);
 	esclen = jsonescapelen(origbuf, origlen, paranoid);
 	if (esclen < 0)
-		return NULL;  /* unsupported char found */
+		return NULL;  /* unsupported char found or overflow */
 	if (origlen == esclen) {
 		Py_INCREF(origstr);
 		return origstr;