Mercurial > hg
diff contrib/python-zstandard/make_cffi.py @ 30822:b54a2984cdd4
zstd: vendor python-zstandard 0.6.0
Commit 63c68d6f5fc8de4afd9bde81b13b537beb4e47e8 from
https://github.com/indygreg/python-zstandard is imported without
modifications (other than removing unwanted files).
This includes minor performance and feature improvements. It also
changes the vendored zstd library from 1.1.1 to 1.1.2.
# no-check-commit
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Sat, 14 Jan 2017 19:41:43 -0800 |
parents | b86a448a2965 |
children | c32454d69b85 |
line wrap: on
line diff
--- a/contrib/python-zstandard/make_cffi.py Sat Jan 14 20:05:15 2017 +0530 +++ b/contrib/python-zstandard/make_cffi.py Sat Jan 14 19:41:43 2017 -0800 @@ -7,7 +7,10 @@ from __future__ import absolute_import import cffi +import distutils.ccompiler import os +import subprocess +import tempfile HERE = os.path.abspath(os.path.dirname(__file__)) @@ -20,10 +23,8 @@ 'common/zstd_common.c', 'compress/fse_compress.c', 'compress/huf_compress.c', - 'compress/zbuff_compress.c', 'compress/zstd_compress.c', 'decompress/huf_decompress.c', - 'decompress/zbuff_decompress.c', 'decompress/zstd_decompress.c', 'dictBuilder/divsufsort.c', 'dictBuilder/zdict.c', @@ -37,74 +38,71 @@ 'zstd/dictBuilder', )] +# cffi can't parse some of the primitives in zstd.h. So we invoke the +# preprocessor and feed its output into cffi. +compiler = distutils.ccompiler.new_compiler() + +# Needed for MSVC. +if hasattr(compiler, 'initialize'): + compiler.initialize() + +# Distutils doesn't set compiler.preprocessor, so invoke the preprocessor +# manually. +if compiler.compiler_type == 'unix': + args = list(compiler.executables['compiler']) + args.extend([ + '-E', + '-DZSTD_STATIC_LINKING_ONLY', + ]) +elif compiler.compiler_type == 'msvc': + args = [compiler.cc] + args.extend([ + '/EP', + '/DZSTD_STATIC_LINKING_ONLY', + ]) +else: + raise Exception('unsupported compiler type: %s' % compiler.compiler_type) + +# zstd.h includes <stddef.h>, which is also included by cffi's boilerplate. +# This can lead to duplicate declarations. So we strip this include from the +# preprocessor invocation. + with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: - zstd_h = fh.read() + lines = [l for l in fh if not l.startswith(b'#include <stddef.h>')] + +fd, input_file = tempfile.mkstemp(suffix='.h') +os.write(fd, b''.join(lines)) +os.close(fd) + +args.append(input_file) + +try: + process = subprocess.Popen(args, stdout=subprocess.PIPE) + output = process.communicate()[0] + ret = process.poll() + if ret: + raise Exception('preprocessor exited with error') +finally: + os.unlink(input_file) + +def normalize_output(): + lines = [] + for line in output.splitlines(): + # CFFI's parser doesn't like __attribute__ on UNIX compilers. + if line.startswith(b'__attribute__ ((visibility ("default"))) '): + line = line[len(b'__attribute__ ((visibility ("default"))) '):] + + lines.append(line) + + return b'\n'.join(lines) ffi = cffi.FFI() ffi.set_source('_zstd_cffi', ''' -/* needed for typedefs like U32 references in zstd.h */ -#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" -''', - sources=SOURCES, include_dirs=INCLUDE_DIRS) - -# Rather than define the API definitions from zstd.h inline, munge the -# source in a way that cdef() will accept. -lines = zstd_h.splitlines() -lines = [l.rstrip() for l in lines if l.strip()] - -# Strip preprocessor directives - they aren't important for our needs. -lines = [l for l in lines - if not l.startswith((b'#if', b'#else', b'#endif', b'#include'))] - -# Remove extern C block -lines = [l for l in lines if l not in (b'extern "C" {', b'}')] - -# The version #defines don't parse and aren't necessary. Strip them. -lines = [l for l in lines if not l.startswith(( - b'#define ZSTD_H_235446', - b'#define ZSTD_LIB_VERSION', - b'#define ZSTD_QUOTE', - b'#define ZSTD_EXPAND_AND_QUOTE', - b'#define ZSTD_VERSION_STRING', - b'#define ZSTD_VERSION_NUMBER'))] +''', sources=SOURCES, include_dirs=INCLUDE_DIRS) -# The C parser also doesn't like some constant defines referencing -# other constants. -# TODO we pick the 64-bit constants here. We should assert somewhere -# we're compiling for 64-bit. -def fix_constants(l): - if l.startswith(b'#define ZSTD_WINDOWLOG_MAX '): - return b'#define ZSTD_WINDOWLOG_MAX 27' - elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): - return b'#define ZSTD_CHAINLOG_MAX 28' - elif l.startswith(b'#define ZSTD_HASHLOG_MAX '): - return b'#define ZSTD_HASHLOG_MAX 27' - elif l.startswith(b'#define ZSTD_CHAINLOG_MAX '): - return b'#define ZSTD_CHAINLOG_MAX 28' - elif l.startswith(b'#define ZSTD_CHAINLOG_MIN '): - return b'#define ZSTD_CHAINLOG_MIN 6' - elif l.startswith(b'#define ZSTD_SEARCHLOG_MAX '): - return b'#define ZSTD_SEARCHLOG_MAX 26' - elif l.startswith(b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX '): - return b'#define ZSTD_BLOCKSIZE_ABSOLUTEMAX 131072' - else: - return l -lines = map(fix_constants, lines) - -# ZSTDLIB_API isn't handled correctly. Strip it. -lines = [l for l in lines if not l.startswith(b'# define ZSTDLIB_API')] -def strip_api(l): - if l.startswith(b'ZSTDLIB_API '): - return l[len(b'ZSTDLIB_API '):] - else: - return l -lines = map(strip_api, lines) - -source = b'\n'.join(lines) -ffi.cdef(source.decode('latin1')) - +ffi.cdef(normalize_output().decode('latin1')) if __name__ == '__main__': ffi.compile()