comparison contrib/python-zstandard/zstd.c @ 31796:e0dc40530c5a

zstd: vendor python-zstandard 0.8.0 Commit 81e1f5bbf1fc54808649562d3ed829730765c540 from https://github.com/indygreg/python-zstandard is imported without modifications (other than removing unwanted files). Updates relevant to Mercurial include: * Support for multi-threaded compression (we can use this for bundle and wire protocol compression). * APIs for batch compression and decompression operations using multiple threads and optimal memory allocation mechanism. (Can be useful for revlog perf improvements.) * A ``BufferWithSegments`` type that models a single memory buffer containing N discrete items of known lengths. This type can be used for very efficient 0-copy data operations. # no-check-commit
author Gregory Szorc <gregory.szorc@gmail.com>
date Sat, 01 Apr 2017 15:24:03 -0700
parents c32454d69b85
children 39d36c2db68e
comparison
equal deleted inserted replaced
31795:2b130e26c3a4 31796:e0dc40530c5a
6 * of the BSD license. See the LICENSE file for details. 6 * of the BSD license. See the LICENSE file for details.
7 */ 7 */
8 8
9 /* A Python C extension for Zstandard. */ 9 /* A Python C extension for Zstandard. */
10 10
11 #if defined(_WIN32)
12 #define WIN32_LEAN_AND_MEAN
13 #include <Windows.h>
14 #endif
15
11 #include "python-zstandard.h" 16 #include "python-zstandard.h"
12 17
13 PyObject *ZstdError; 18 PyObject *ZstdError;
14 19
15 PyDoc_STRVAR(estimate_compression_context_size__doc__, 20 PyDoc_STRVAR(estimate_compression_context_size__doc__,
47 "A compression dictionary of size ``dict_size`` will be created from the\n" 52 "A compression dictionary of size ``dict_size`` will be created from the\n"
48 "iterable of samples provided by ``samples``.\n" 53 "iterable of samples provided by ``samples``.\n"
49 "\n" 54 "\n"
50 "The raw dictionary content will be returned\n"); 55 "The raw dictionary content will be returned\n");
51 56
57 PyDoc_STRVAR(train_cover_dictionary__doc__,
58 "train_cover_dictionary(dict_size, samples, k=None, d=None, notifications=0, dict_id=0, level=0)\n"
59 "\n"
60 "Train a dictionary from sample data using the COVER algorithm.\n"
61 "\n"
62 "This behaves like ``train_dictionary()`` except a different algorithm is\n"
63 "used to create the dictionary. The algorithm has 2 parameters: ``k`` and\n"
64 "``d``. These control the *segment size* and *dmer size*. A reasonable range\n"
65 "for ``k`` is ``[16, 2048+]``. A reasonable range for ``d`` is ``[6, 16]``.\n"
66 "``d`` must be less than or equal to ``k``.\n"
67 );
68
52 static char zstd_doc[] = "Interface to zstandard"; 69 static char zstd_doc[] = "Interface to zstandard";
53 70
54 static PyMethodDef zstd_methods[] = { 71 static PyMethodDef zstd_methods[] = {
72 /* TODO remove since it is a method on CompressionParameters. */
55 { "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size, 73 { "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size,
56 METH_VARARGS, estimate_compression_context_size__doc__ }, 74 METH_VARARGS, estimate_compression_context_size__doc__ },
57 { "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size, 75 { "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size,
58 METH_NOARGS, estimate_decompression_context_size__doc__ }, 76 METH_NOARGS, estimate_decompression_context_size__doc__ },
59 { "get_compression_parameters", (PyCFunction)get_compression_parameters, 77 { "get_compression_parameters", (PyCFunction)get_compression_parameters,
60 METH_VARARGS, get_compression_parameters__doc__ }, 78 METH_VARARGS, get_compression_parameters__doc__ },
61 { "get_frame_parameters", (PyCFunction)get_frame_parameters, 79 { "get_frame_parameters", (PyCFunction)get_frame_parameters,
62 METH_VARARGS, get_frame_parameters__doc__ }, 80 METH_VARARGS, get_frame_parameters__doc__ },
63 { "train_dictionary", (PyCFunction)train_dictionary, 81 { "train_dictionary", (PyCFunction)train_dictionary,
64 METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ }, 82 METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ },
83 { "train_cover_dictionary", (PyCFunction)train_cover_dictionary,
84 METH_VARARGS | METH_KEYWORDS, train_cover_dictionary__doc__ },
65 { NULL, NULL } 85 { NULL, NULL }
66 }; 86 };
67 87
88 void bufferutil_module_init(PyObject* mod);
68 void compressobj_module_init(PyObject* mod); 89 void compressobj_module_init(PyObject* mod);
69 void compressor_module_init(PyObject* mod); 90 void compressor_module_init(PyObject* mod);
70 void compressionparams_module_init(PyObject* mod); 91 void compressionparams_module_init(PyObject* mod);
71 void constants_module_init(PyObject* mod); 92 void constants_module_init(PyObject* mod);
72 void dictparams_module_init(PyObject* mod);
73 void compressiondict_module_init(PyObject* mod); 93 void compressiondict_module_init(PyObject* mod);
74 void compressionwriter_module_init(PyObject* mod); 94 void compressionwriter_module_init(PyObject* mod);
75 void compressoriterator_module_init(PyObject* mod); 95 void compressoriterator_module_init(PyObject* mod);
76 void decompressor_module_init(PyObject* mod); 96 void decompressor_module_init(PyObject* mod);
77 void decompressobj_module_init(PyObject* mod); 97 void decompressobj_module_init(PyObject* mod);
98 if (ZSTD_VERSION_NUMBER != 10103 || ZSTD_versionNumber() != 10103) { 118 if (ZSTD_VERSION_NUMBER != 10103 || ZSTD_versionNumber() != 10103) {
99 PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version"); 119 PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
100 return; 120 return;
101 } 121 }
102 122
123 bufferutil_module_init(m);
103 compressionparams_module_init(m); 124 compressionparams_module_init(m);
104 dictparams_module_init(m);
105 compressiondict_module_init(m); 125 compressiondict_module_init(m);
106 compressobj_module_init(m); 126 compressobj_module_init(m);
107 compressor_module_init(m); 127 compressor_module_init(m);
108 compressionwriter_module_init(m); 128 compressionwriter_module_init(m);
109 compressoriterator_module_init(m); 129 compressoriterator_module_init(m);
141 if (m) { 161 if (m) {
142 zstd_module_init(m); 162 zstd_module_init(m);
143 } 163 }
144 } 164 }
145 #endif 165 #endif
166
167 /* Attempt to resolve the number of CPUs in the system. */
168 int cpu_count() {
169 int count = 0;
170
171 #if defined(_WIN32)
172 SYSTEM_INFO si;
173 si.dwNumberOfProcessors = 0;
174 GetSystemInfo(&si);
175 count = si.dwNumberOfProcessors;
176 #elif defined(__APPLE__)
177 int num;
178 size_t size = sizeof(int);
179
180 if (0 == sysctlbyname("hw.logicalcpu", &num, &size, NULL, 0)) {
181 count = num;
182 }
183 #elif defined(__linux__)
184 count = sysconf(_SC_NPROCESSORS_ONLN);
185 #elif defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__)
186 int mib[2];
187 size_t len = sizeof(count);
188 mib[0] = CTL_HW;
189 mib[1] = HW_NCPU;
190 if (0 != sysctl(mib, 2, &count, &len, NULL, 0)) {
191 count = 0;
192 }
193 #elif defined(__hpux)
194 count = mpctl(MPC_GETNUMSPUS, NULL, NULL);
195 #endif
196
197 return count;
198 }
199
200 size_t roundpow2(size_t i) {
201 i--;
202 i |= i >> 1;
203 i |= i >> 2;
204 i |= i >> 4;
205 i |= i >> 8;
206 i |= i >> 16;
207 i++;
208
209 return i;
210 }