Mercurial > hg
comparison contrib/python-zstandard/make_cffi.py @ 30822:b54a2984cdd4
zstd: vendor python-zstandard 0.6.0
Commit 63c68d6f5fc8de4afd9bde81b13b537beb4e47e8 from
https://github.com/indygreg/python-zstandard is imported without
modifications (other than removing unwanted files).
This includes minor performance and feature improvements. It also
changes the vendored zstd library from 1.1.1 to 1.1.2.
# no-check-commit
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Sat, 14 Jan 2017 19:41:43 -0800 |
parents | b86a448a2965 |
children | c32454d69b85 |
comparison
equal
deleted
inserted
replaced
30821:7005c03f7387 | 30822:b54a2984cdd4 |
---|---|
5 # of the BSD license. See the LICENSE file for details. | 5 # of the BSD license. See the LICENSE file for details. |
6 | 6 |
7 from __future__ import absolute_import | 7 from __future__ import absolute_import |
8 | 8 |
9 import cffi | 9 import cffi |
10 import distutils.ccompiler | |
10 import os | 11 import os |
12 import subprocess | |
13 import tempfile | |
11 | 14 |
12 | 15 |
13 HERE = os.path.abspath(os.path.dirname(__file__)) | 16 HERE = os.path.abspath(os.path.dirname(__file__)) |
14 | 17 |
15 SOURCES = ['zstd/%s' % p for p in ( | 18 SOURCES = ['zstd/%s' % p for p in ( |
18 'common/fse_decompress.c', | 21 'common/fse_decompress.c', |
19 'common/xxhash.c', | 22 'common/xxhash.c', |
20 'common/zstd_common.c', | 23 'common/zstd_common.c', |
21 'compress/fse_compress.c', | 24 'compress/fse_compress.c', |
22 'compress/huf_compress.c', | 25 'compress/huf_compress.c', |
23 'compress/zbuff_compress.c', | |
24 'compress/zstd_compress.c', | 26 'compress/zstd_compress.c', |
25 'decompress/huf_decompress.c', | 27 'decompress/huf_decompress.c', |
26 'decompress/zbuff_decompress.c', | |
27 'decompress/zstd_decompress.c', | 28 'decompress/zstd_decompress.c', |
28 'dictBuilder/divsufsort.c', | 29 'dictBuilder/divsufsort.c', |
29 'dictBuilder/zdict.c', | 30 'dictBuilder/zdict.c', |
30 )] | 31 )] |
31 | 32 |
35 'zstd/compress', | 36 'zstd/compress', |
36 'zstd/decompress', | 37 'zstd/decompress', |
37 'zstd/dictBuilder', | 38 'zstd/dictBuilder', |
38 )] | 39 )] |
39 | 40 |
41 # cffi can't parse some of the primitives in zstd.h. So we invoke the | |
42 # preprocessor and feed its output into cffi. | |
43 compiler = distutils.ccompiler.new_compiler() | |
44 | |
45 # Needed for MSVC. | |
46 if hasattr(compiler, 'initialize'): | |
47 compiler.initialize() | |
48 | |
49 # Distutils doesn't set compiler.preprocessor, so invoke the preprocessor | |
50 # manually. | |
51 if compiler.compiler_type == 'unix': | |
52 args = list(compiler.executables['compiler']) | |
53 args.extend([ | |
54 '-E', | |
55 '-DZSTD_STATIC_LINKING_ONLY', | |
56 ]) | |
57 elif compiler.compiler_type == 'msvc': | |
58 args = [compiler.cc] | |
59 args.extend([ | |
60 '/EP', | |
61 '/DZSTD_STATIC_LINKING_ONLY', | |
62 ]) | |
63 else: | |
64 raise Exception('unsupported compiler type: %s' % compiler.compiler_type) | |
65 | |
66 # zstd.h includes <stddef.h>, which is also included by cffi's boilerplate. | |
67 # This can lead to duplicate declarations. So we strip this include from the | |
68 # preprocessor invocation. | |
69 | |
40 with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: | 70 with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: |
41 zstd_h = fh.read() | 71 lines = [l for l in fh if not l.startswith(b'#include <stddef.h>')] |
72 | |
73 fd, input_file = tempfile.mkstemp(suffix='.h') | |
74 os.write(fd, b''.join(lines)) | |
75 os.close(fd) | |
76 | |
77 args.append(input_file) | |
78 | |
79 try: | |
80 process = subprocess.Popen(args, stdout=subprocess.PIPE) | |
81 output = process.communicate()[0] | |
82 ret = process.poll() | |
83 if ret: | |
84 raise Exception('preprocessor exited with error') | |
85 finally: | |
86 os.unlink(input_file) | |
87 | |
88 def normalize_output(): | |
89 lines = [] | |
90 for line in output.splitlines(): | |
91 # CFFI's parser doesn't like __attribute__ on UNIX compilers. | |
92 if line.startswith(b'__attribute__ ((visibility ("default"))) '): | |
93 line = line[len(b'__attribute__ ((visibility ("default"))) '):] | |
94 | |
95 lines.append(line) | |
96 | |
97 return b'\n'.join(lines) | |
42 | 98 |
43 ffi = cffi.FFI() | 99 ffi = cffi.FFI() |
44 ffi.set_source('_zstd_cffi', ''' | 100 ffi.set_source('_zstd_cffi', ''' |
45 /* needed for typedefs like U32 references in zstd.h */ | |
46 #include "mem.h" | |
47 #define ZSTD_STATIC_LINKING_ONLY | 101 #define ZSTD_STATIC_LINKING_ONLY |
48 #include "zstd.h" | 102 #include "zstd.h" |
49 ''', | 103 ''', sources=SOURCES, include_dirs=INCLUDE_DIRS) |
50 sources=SOURCES, include_dirs=INCLUDE_DIRS) | |
51 | 104 |
52 # Rather than define the API definitions from zstd.h inline, munge the | 105 ffi.cdef(normalize_output().decode('latin1')) |
53 # source in a way that cdef() will accept. | |
54 lines = zstd_h.splitlines() | |
55 lines = [l.rstrip() for l in lines if l.strip()] | |
56 | |
57 # Strip preprocessor directives - they aren't important for our needs. | |
58 lines = [l for l in lines | |
59 if not l.startswith((b'#if', b'#else', b'#endif', b'#include'))] | |
60 | |
61 # Remove extern C block | |
62 lines = [l for l in lines if l not in (b'extern "C" {', b'}')] | |
63 | |
64 # The version #defines don't parse and aren't necessary. Strip them. | |
65 lines = [l for l in lines if not l.startswith(( | |
66 b'#define ZSTD_H_235446', | |
67 b'#define ZSTD_LIB_VERSION', | |
68 b'#define ZSTD_QUOTE', | |
69 b'#define ZSTD_EXPAND_AND_QUOTE', | |
70 b'#define ZSTD_VERSION_STRING', | |
71 b'#define ZSTD_VERSION_NUMBER'))] | |
72 | |
73 # The C parser also doesn't like some constant defines referencing | |
74 # other constants. | |
75 # TODO we pick the 64-bit constants here. We should assert somewhere | |
76 # we're compiling for 64-bit. | |
77 def fix_constants(l): | |
78 if l.startswith(b'#define ZSTD_WINDOWLOG_MAX '): | |
79 return b'#define ZSTD_WINDOWLOG_MAX 27' | |
80 elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): | |
81 return b'#define ZSTD_CHAINLOG_MAX 28' | |
82 elif l.startswith(b'#define ZSTD_HASHLOG_MAX '): | |
83 return b'#define ZSTD_HASHLOG_MAX 27' | |
84 elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): | |
85 return b'#define ZSTD_CHAINLOG_MAX 28' | |
86 elif l.startswith(b'#define ZSTD_CHAINLOG_MIN '): | |
87 return b'#define ZSTD_CHAINLOG_MIN 6' | |
88 elif l.startswith(b'#define ZSTD_SEARCHLOG_MAX '): | |
89 return b'#define ZSTD_SEARCHLOG_MAX 26' | |
90 elif l.startswith(b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX '): | |
91 return b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX 131072' | |
92 else: | |
93 return l | |
94 lines = map(fix_constants, lines) | |
95 | |
96 # ZSTDLIB_API isn't handled correctly. Strip it. | |
97 lines = [l for l in lines if not l.startswith(b'# define ZSTDLIB_API')] | |
98 def strip_api(l): | |
99 if l.startswith(b'ZSTDLIB_API '): | |
100 return l[len(b'ZSTDLIB_API '):] | |
101 else: | |
102 return l | |
103 lines = map(strip_api, lines) | |
104 | |
105 source = b'\n'.join(lines) | |
106 ffi.cdef(source.decode('latin1')) | |
107 | |
108 | 106 |
109 if __name__ == '__main__': | 107 if __name__ == '__main__': |
110 ffi.compile() | 108 ffi.compile() |