mercurial/cext/charencode.c
author Yuya Nishihara <yuya@tcha.org>
Mon, 14 Aug 2017 13:35:26 +0900
changeset 33833 4ba863c88135
parent 33780 a22339d389d4
child 33943 2c37f9dabc32
permissions -rw-r--r--
cext: move PyInt macros to charencode.c properly Python3 build was broken at e9996bd7203f.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     1
/*
33778
e9996bd7203f cext: split character encoding functions to new compilation unit
Yuya Nishihara <yuya@tcha.org>
parents: 33777
diff changeset
     2
 charencode.c - miscellaneous character encoding
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     3
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     4
 Copyright 2008 Matt Mackall <mpm@selenic.com> and others
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     5
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     6
 This software may be used and distributed according to the terms of
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     7
 the GNU General Public License, incorporated herein by reference.
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     8
*/
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
     9
33780
a22339d389d4 cext: modernize charencode.c to use Py_ssize_t
Yuya Nishihara <yuya@tcha.org>
parents: 33779
diff changeset
    10
#define PY_SSIZE_T_CLEAN
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    11
#include <Python.h>
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    12
33779
0f4ac3b6dee4 cext: factor out header for charencode.c
Yuya Nishihara <yuya@tcha.org>
parents: 33778
diff changeset
    13
#include "charencode.h"
11361
3de3d670d2b6 parsers.c: Added support for py3k.
Renato Cunha <renatoc@gmail.com>
parents: 10449
diff changeset
    14
#include "util.h"
20742
3681de20b0a7 parsers: fail fast if Python has wrong minor version (issue4110)
Chris Jerdonek <chris.jerdonek@gmail.com>
parents: 20555
diff changeset
    15
33833
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    16
#ifdef IS_PY3K
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    17
/* The mapping of Python types is meant to be temporary to get Python
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    18
 * 3 to compile. We should remove this once Python 3 support is fully
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    19
 * supported and proper types are used in the extensions themselves. */
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    20
#define PyInt_Type PyLong_Type
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    21
#define PyInt_AS_LONG PyLong_AS_LONG
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    22
#endif
4ba863c88135 cext: move PyInt macros to charencode.c properly
Yuya Nishihara <yuya@tcha.org>
parents: 33780
diff changeset
    23
32424
7640584e697c cext: mark constant variables
Yuya Nishihara <yuya@tcha.org>
parents: 32422
diff changeset
    24
static const char lowertable[128] = {
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    25
	'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    26
	'\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    27
	'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    28
	'\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    29
	'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    30
	'\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    31
	'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    32
	'\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    33
	'\x40',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    34
	        '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67', /* A-G */
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    35
	'\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f', /* H-O */
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    36
	'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77', /* P-W */
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    37
	'\x78', '\x79', '\x7a',                                         /* X-Z */
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    38
	                        '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    39
	'\x60', '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    40
	'\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    41
	'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77',
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    42
	'\x78', '\x79', '\x7a', '\x7b', '\x7c', '\x7d', '\x7e', '\x7f'
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    43
};
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    44
32424
7640584e697c cext: mark constant variables
Yuya Nishihara <yuya@tcha.org>
parents: 32422
diff changeset
    45
static const char uppertable[128] = {
24577
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    46
	'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    47
	'\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    48
	'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    49
	'\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    50
	'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    51
	'\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    52
	'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    53
	'\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    54
	'\x40', '\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    55
	'\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    56
	'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    57
	'\x58', '\x59', '\x5a', '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    58
	'\x60',
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    59
		'\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47', /* a-g */
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    60
	'\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f', /* h-o */
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    61
	'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57', /* p-w */
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    62
	'\x58', '\x59', '\x5a', 					/* x-z */
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    63
				'\x7b', '\x7c', '\x7d', '\x7e', '\x7f'
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    64
};
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
    65
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    66
/*
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    67
 * Turn a hex-encoded string into binary.
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    68
 */
33780
a22339d389d4 cext: modernize charencode.c to use Py_ssize_t
Yuya Nishihara <yuya@tcha.org>
parents: 33779
diff changeset
    69
PyObject *unhexlify(const char *str, Py_ssize_t len)
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    70
{
7092
fb3fc27617a2 parsers: speed up hex decoding for manifests
Matt Mackall <mpm@selenic.com>
parents: 7091
diff changeset
    71
	PyObject *ret;
6395
3f0294536b24 fix const annotation warning
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 6389
diff changeset
    72
	char *d;
33780
a22339d389d4 cext: modernize charencode.c to use Py_ssize_t
Yuya Nishihara <yuya@tcha.org>
parents: 33779
diff changeset
    73
	Py_ssize_t i;
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    74
11361
3de3d670d2b6 parsers.c: Added support for py3k.
Renato Cunha <renatoc@gmail.com>
parents: 10449
diff changeset
    75
	ret = PyBytes_FromStringAndSize(NULL, len / 2);
3de3d670d2b6 parsers.c: Added support for py3k.
Renato Cunha <renatoc@gmail.com>
parents: 10449
diff changeset
    76
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    77
	if (!ret)
7092
fb3fc27617a2 parsers: speed up hex decoding for manifests
Matt Mackall <mpm@selenic.com>
parents: 7091
diff changeset
    78
		return NULL;
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    79
11361
3de3d670d2b6 parsers.c: Added support for py3k.
Renato Cunha <renatoc@gmail.com>
parents: 10449
diff changeset
    80
	d = PyBytes_AsString(ret);
3de3d670d2b6 parsers.c: Added support for py3k.
Renato Cunha <renatoc@gmail.com>
parents: 10449
diff changeset
    81
16617
4fb16743049d parsers: change the type signature of hexdigit
Bryan O'Sullivan <bryano@fb.com>
parents: 16616
diff changeset
    82
	for (i = 0; i < len;) {
4fb16743049d parsers: change the type signature of hexdigit
Bryan O'Sullivan <bryano@fb.com>
parents: 16616
diff changeset
    83
		int hi = hexdigit(str, i++);
4fb16743049d parsers: change the type signature of hexdigit
Bryan O'Sullivan <bryano@fb.com>
parents: 16616
diff changeset
    84
		int lo = hexdigit(str, i++);
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    85
		*d++ = (hi << 4) | lo;
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    86
	}
7091
12b35ae03365 parsers: clean up whitespace
Matt Mackall <mpm@selenic.com>
parents: 6395
diff changeset
    87
6389
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    88
	return ret;
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    89
}
0231f763ebc8 manifest: improve parsing performance by 8x via a new C extension
Bryan O'Sullivan <bos@serpentine.com>
parents:
diff changeset
    90
24576
fe173106e7fe parsers: make _asciilower a generic _asciitransform function
Siddharth Agarwal <sid0@fb.com>
parents: 24575
diff changeset
    91
static inline PyObject *_asciitransform(PyObject *str_obj,
24606
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
    92
					const char table[128],
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
    93
					PyObject *fallback_fn)
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    94
{
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    95
	char *str, *newstr;
24574
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
    96
	Py_ssize_t i, len;
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    97
	PyObject *newobj = NULL;
24575
a62e957413f7 parsers._asciilower: use an explicit return object
Siddharth Agarwal <sid0@fb.com>
parents: 24574
diff changeset
    98
	PyObject *ret = NULL;
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
    99
24574
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   100
	str = PyBytes_AS_STRING(str_obj);
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   101
	len = PyBytes_GET_SIZE(str_obj);
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   102
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   103
	newobj = PyBytes_FromStringAndSize(NULL, len);
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   104
	if (!newobj)
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   105
		goto quit;
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   106
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   107
	newstr = PyBytes_AS_STRING(newobj);
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   108
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   109
	for (i = 0; i < len; i++) {
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   110
		char c = str[i];
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   111
		if (c & 0x80) {
24606
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   112
			if (fallback_fn != NULL) {
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   113
				ret = PyObject_CallFunctionObjArgs(fallback_fn,
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   114
					str_obj, NULL);
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   115
			} else {
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   116
				PyObject *err = PyUnicodeDecodeError_Create(
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   117
					"ascii", str, len, i, (i + 1),
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   118
					"unexpected code byte");
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   119
				PyErr_SetObject(PyExc_UnicodeDecodeError, err);
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   120
				Py_XDECREF(err);
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   121
			}
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   122
			goto quit;
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   123
		}
24576
fe173106e7fe parsers: make _asciilower a generic _asciitransform function
Siddharth Agarwal <sid0@fb.com>
parents: 24575
diff changeset
   124
		newstr[i] = table[(unsigned char)c];
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   125
	}
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   126
24575
a62e957413f7 parsers._asciilower: use an explicit return object
Siddharth Agarwal <sid0@fb.com>
parents: 24574
diff changeset
   127
	ret = newobj;
a62e957413f7 parsers._asciilower: use an explicit return object
Siddharth Agarwal <sid0@fb.com>
parents: 24574
diff changeset
   128
	Py_INCREF(ret);
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   129
quit:
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   130
	Py_XDECREF(newobj);
24575
a62e957413f7 parsers._asciilower: use an explicit return object
Siddharth Agarwal <sid0@fb.com>
parents: 24574
diff changeset
   131
	return ret;
22778
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   132
}
80f2b63dd83a parsers: add a function to efficiently lowercase ASCII strings
Siddharth Agarwal <sid0@fb.com>
parents: 22604
diff changeset
   133
33778
e9996bd7203f cext: split character encoding functions to new compilation unit
Yuya Nishihara <yuya@tcha.org>
parents: 33777
diff changeset
   134
PyObject *asciilower(PyObject *self, PyObject *args)
24574
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   135
{
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   136
	PyObject *str_obj;
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   137
	if (!PyArg_ParseTuple(args, "O!:asciilower", &PyBytes_Type, &str_obj))
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   138
		return NULL;
24606
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   139
	return _asciitransform(str_obj, lowertable, NULL);
24574
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   140
}
e97a00bf18ae parsers: factor out most of asciilower into an internal function
Siddharth Agarwal <sid0@fb.com>
parents: 24499
diff changeset
   141
33778
e9996bd7203f cext: split character encoding functions to new compilation unit
Yuya Nishihara <yuya@tcha.org>
parents: 33777
diff changeset
   142
PyObject *asciiupper(PyObject *self, PyObject *args)
24577
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
   143
{
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
   144
	PyObject *str_obj;
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
   145
	if (!PyArg_ParseTuple(args, "O!:asciiupper", &PyBytes_Type, &str_obj))
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
   146
		return NULL;
24606
e4a733c34bc6 parsers._asciitransform: also accept a fallback function
Siddharth Agarwal <sid0@fb.com>
parents: 24577
diff changeset
   147
	return _asciitransform(str_obj, uppertable, NULL);
24577
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
   148
}
bf55df007535 parsers: introduce an asciiupper function
Siddharth Agarwal <sid0@fb.com>
parents: 24576
diff changeset
   149
33778
e9996bd7203f cext: split character encoding functions to new compilation unit
Yuya Nishihara <yuya@tcha.org>
parents: 33777
diff changeset
   150
PyObject *make_file_foldmap(PyObject *self, PyObject *args)
24609
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   151
{
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   152
	PyObject *dmap, *spec_obj, *normcase_fallback;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   153
	PyObject *file_foldmap = NULL;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   154
	enum normcase_spec spec;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   155
	PyObject *k, *v;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   156
	dirstateTupleObject *tuple;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   157
	Py_ssize_t pos = 0;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   158
	const char *table;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   159
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   160
	if (!PyArg_ParseTuple(args, "O!O!O!:make_file_foldmap",
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   161
			      &PyDict_Type, &dmap,
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   162
			      &PyInt_Type, &spec_obj,
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   163
			      &PyFunction_Type, &normcase_fallback))
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   164
		goto quit;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   165
24622
1e05f11619bb parsers.c: avoid implicit conversion loses integer precision warning
André Sintzoff <andre.sintzoff@gmail.com>
parents: 24609
diff changeset
   166
	spec = (int)PyInt_AS_LONG(spec_obj);
24609
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   167
	switch (spec) {
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   168
	case NORMCASE_LOWER:
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   169
		table = lowertable;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   170
		break;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   171
	case NORMCASE_UPPER:
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   172
		table = uppertable;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   173
		break;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   174
	case NORMCASE_OTHER:
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   175
		table = NULL;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   176
		break;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   177
	default:
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   178
		PyErr_SetString(PyExc_TypeError, "invalid normcasespec");
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   179
		goto quit;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   180
	}
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   181
25583
ce64c9ab19f2 parsers: factor out code to create a presized dict
Siddharth Agarwal <sid0@fb.com>
parents: 25582
diff changeset
   182
	/* Add some more entries to deal with additions outside this
ce64c9ab19f2 parsers: factor out code to create a presized dict
Siddharth Agarwal <sid0@fb.com>
parents: 25582
diff changeset
   183
	   function. */
ce64c9ab19f2 parsers: factor out code to create a presized dict
Siddharth Agarwal <sid0@fb.com>
parents: 25582
diff changeset
   184
	file_foldmap = _dict_new_presized((PyDict_Size(dmap) / 10) * 11);
24609
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   185
	if (file_foldmap == NULL)
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   186
		goto quit;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   187
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   188
	while (PyDict_Next(dmap, &pos, &k, &v)) {
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   189
		if (!dirstate_tuple_check(v)) {
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   190
			PyErr_SetString(PyExc_TypeError,
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   191
					"expected a dirstate tuple");
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   192
			goto quit;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   193
		}
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   194
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   195
		tuple = (dirstateTupleObject *)v;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   196
		if (tuple->state != 'r') {
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   197
			PyObject *normed;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   198
			if (table != NULL) {
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   199
				normed = _asciitransform(k, table,
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   200
					normcase_fallback);
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   201
			} else {
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   202
				normed = PyObject_CallFunctionObjArgs(
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   203
					normcase_fallback, k, NULL);
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   204
			}
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   205
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   206
			if (normed == NULL)
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   207
				goto quit;
26049
b1634b7804c7 parsers: correctly decref normed value after PyDict_SetItem
Augie Fackler <augie@google.com>
parents: 26048
diff changeset
   208
			if (PyDict_SetItem(file_foldmap, normed, k) == -1) {
b1634b7804c7 parsers: correctly decref normed value after PyDict_SetItem
Augie Fackler <augie@google.com>
parents: 26048
diff changeset
   209
				Py_DECREF(normed);
24609
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   210
				goto quit;
26049
b1634b7804c7 parsers: correctly decref normed value after PyDict_SetItem
Augie Fackler <augie@google.com>
parents: 26048
diff changeset
   211
			}
b1634b7804c7 parsers: correctly decref normed value after PyDict_SetItem
Augie Fackler <augie@google.com>
parents: 26048
diff changeset
   212
			Py_DECREF(normed);
24609
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   213
		}
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   214
	}
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   215
	return file_foldmap;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   216
quit:
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   217
	Py_XDECREF(file_foldmap);
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   218
	return NULL;
670aaee7931c parsers: add a C function to create a file foldmap
Siddharth Agarwal <sid0@fb.com>
parents: 24606
diff changeset
   219
}