Mercurial > hg
diff contrib/python-zstandard/zstandard/cffi.py @ 43994:de7838053207
zstandard: vendor python-zstandard 0.13.0
Version 0.13.0 of the package was just released. It contains
an upgraded zstd C library which can result in some performance
wins, official support for Python 3.8, and a blackened code base.
There were no meaningful code or functionality changes in this
release of python-zstandard: just reformatting and an upgraded
zstd library version. So the diff seems much larger than what it
is.
Files were added without modifications.
The clang-format-ignorelist file was updated to reflect a new
header file in the zstd distribution.
# no-check-commit because 3rd party code has different style guidelines
Differential Revision: https://phab.mercurial-scm.org/D7770
| author | Gregory Szorc <gregory.szorc@gmail.com> |
|---|---|
| date | Sat, 28 Dec 2019 09:55:45 -0800 |
| parents | 69de49c4e39c |
| children | 5e84a96d865b |
line wrap: on
line diff
--- a/contrib/python-zstandard/zstandard/cffi.py Fri Dec 27 18:54:57 2019 -0500 +++ b/contrib/python-zstandard/zstandard/cffi.py Sat Dec 28 09:55:45 2019 -0800 @@ -14,68 +14,67 @@ #'BufferSegments', #'BufferWithSegments', #'BufferWithSegmentsCollection', - 'CompressionParameters', - 'ZstdCompressionDict', - 'ZstdCompressionParameters', - 'ZstdCompressor', - 'ZstdError', - 'ZstdDecompressor', - 'FrameParameters', - 'estimate_decompression_context_size', - 'frame_content_size', - 'frame_header_size', - 'get_frame_parameters', - 'train_dictionary', - + "CompressionParameters", + "ZstdCompressionDict", + "ZstdCompressionParameters", + "ZstdCompressor", + "ZstdError", + "ZstdDecompressor", + "FrameParameters", + "estimate_decompression_context_size", + "frame_content_size", + "frame_header_size", + "get_frame_parameters", + "train_dictionary", # Constants. - 'FLUSH_BLOCK', - 'FLUSH_FRAME', - 'COMPRESSOBJ_FLUSH_FINISH', - 'COMPRESSOBJ_FLUSH_BLOCK', - 'ZSTD_VERSION', - 'FRAME_HEADER', - 'CONTENTSIZE_UNKNOWN', - 'CONTENTSIZE_ERROR', - 'MAX_COMPRESSION_LEVEL', - 'COMPRESSION_RECOMMENDED_INPUT_SIZE', - 'COMPRESSION_RECOMMENDED_OUTPUT_SIZE', - 'DECOMPRESSION_RECOMMENDED_INPUT_SIZE', - 'DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE', - 'MAGIC_NUMBER', - 'BLOCKSIZELOG_MAX', - 'BLOCKSIZE_MAX', - 'WINDOWLOG_MIN', - 'WINDOWLOG_MAX', - 'CHAINLOG_MIN', - 'CHAINLOG_MAX', - 'HASHLOG_MIN', - 'HASHLOG_MAX', - 'HASHLOG3_MAX', - 'MINMATCH_MIN', - 'MINMATCH_MAX', - 'SEARCHLOG_MIN', - 'SEARCHLOG_MAX', - 'SEARCHLENGTH_MIN', - 'SEARCHLENGTH_MAX', - 'TARGETLENGTH_MIN', - 'TARGETLENGTH_MAX', - 'LDM_MINMATCH_MIN', - 'LDM_MINMATCH_MAX', - 'LDM_BUCKETSIZELOG_MAX', - 'STRATEGY_FAST', - 'STRATEGY_DFAST', - 'STRATEGY_GREEDY', - 'STRATEGY_LAZY', - 'STRATEGY_LAZY2', - 'STRATEGY_BTLAZY2', - 'STRATEGY_BTOPT', - 'STRATEGY_BTULTRA', - 'STRATEGY_BTULTRA2', - 'DICT_TYPE_AUTO', - 'DICT_TYPE_RAWCONTENT', - 'DICT_TYPE_FULLDICT', - 'FORMAT_ZSTD1', - 'FORMAT_ZSTD1_MAGICLESS', + "FLUSH_BLOCK", + "FLUSH_FRAME", + "COMPRESSOBJ_FLUSH_FINISH", + "COMPRESSOBJ_FLUSH_BLOCK", + "ZSTD_VERSION", + "FRAME_HEADER", + "CONTENTSIZE_UNKNOWN", + "CONTENTSIZE_ERROR", + "MAX_COMPRESSION_LEVEL", + "COMPRESSION_RECOMMENDED_INPUT_SIZE", + "COMPRESSION_RECOMMENDED_OUTPUT_SIZE", + "DECOMPRESSION_RECOMMENDED_INPUT_SIZE", + "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE", + "MAGIC_NUMBER", + "BLOCKSIZELOG_MAX", + "BLOCKSIZE_MAX", + "WINDOWLOG_MIN", + "WINDOWLOG_MAX", + "CHAINLOG_MIN", + "CHAINLOG_MAX", + "HASHLOG_MIN", + "HASHLOG_MAX", + "HASHLOG3_MAX", + "MINMATCH_MIN", + "MINMATCH_MAX", + "SEARCHLOG_MIN", + "SEARCHLOG_MAX", + "SEARCHLENGTH_MIN", + "SEARCHLENGTH_MAX", + "TARGETLENGTH_MIN", + "TARGETLENGTH_MAX", + "LDM_MINMATCH_MIN", + "LDM_MINMATCH_MAX", + "LDM_BUCKETSIZELOG_MAX", + "STRATEGY_FAST", + "STRATEGY_DFAST", + "STRATEGY_GREEDY", + "STRATEGY_LAZY", + "STRATEGY_LAZY2", + "STRATEGY_BTLAZY2", + "STRATEGY_BTOPT", + "STRATEGY_BTULTRA", + "STRATEGY_BTULTRA2", + "DICT_TYPE_AUTO", + "DICT_TYPE_RAWCONTENT", + "DICT_TYPE_FULLDICT", + "FORMAT_ZSTD1", + "FORMAT_ZSTD1_MAGICLESS", ] import io @@ -105,10 +104,14 @@ MAX_COMPRESSION_LEVEL = lib.ZSTD_maxCLevel() MAGIC_NUMBER = lib.ZSTD_MAGICNUMBER -FRAME_HEADER = b'\x28\xb5\x2f\xfd' +FRAME_HEADER = b"\x28\xb5\x2f\xfd" CONTENTSIZE_UNKNOWN = lib.ZSTD_CONTENTSIZE_UNKNOWN CONTENTSIZE_ERROR = lib.ZSTD_CONTENTSIZE_ERROR -ZSTD_VERSION = (lib.ZSTD_VERSION_MAJOR, lib.ZSTD_VERSION_MINOR, lib.ZSTD_VERSION_RELEASE) +ZSTD_VERSION = ( + lib.ZSTD_VERSION_MAJOR, + lib.ZSTD_VERSION_MINOR, + lib.ZSTD_VERSION_RELEASE, +) BLOCKSIZELOG_MAX = lib.ZSTD_BLOCKSIZELOG_MAX BLOCKSIZE_MAX = lib.ZSTD_BLOCKSIZE_MAX @@ -165,9 +168,9 @@ # Linux. try: if sys.version_info[0] == 2: - return os.sysconf(b'SC_NPROCESSORS_ONLN') + return os.sysconf(b"SC_NPROCESSORS_ONLN") else: - return os.sysconf(u'SC_NPROCESSORS_ONLN') + return os.sysconf("SC_NPROCESSORS_ONLN") except (AttributeError, ValueError): pass @@ -183,7 +186,8 @@ # Resolves to bytes on Python 2 and 3. We use the string for formatting # into error messages, which will be literal unicode. So convert it to # unicode. - return ffi.string(lib.ZSTD_getErrorName(zresult)).decode('utf-8') + return ffi.string(lib.ZSTD_getErrorName(zresult)).decode("utf-8") + def _make_cctx_params(params): res = lib.ZSTD_createCCtxParams() @@ -221,19 +225,20 @@ return res + class ZstdCompressionParameters(object): @staticmethod def from_level(level, source_size=0, dict_size=0, **kwargs): params = lib.ZSTD_getCParams(level, source_size, dict_size) args = { - 'window_log': 'windowLog', - 'chain_log': 'chainLog', - 'hash_log': 'hashLog', - 'search_log': 'searchLog', - 'min_match': 'minMatch', - 'target_length': 'targetLength', - 'compression_strategy': 'strategy', + "window_log": "windowLog", + "chain_log": "chainLog", + "hash_log": "hashLog", + "search_log": "searchLog", + "min_match": "minMatch", + "target_length": "targetLength", + "compression_strategy": "strategy", } for arg, attr in args.items(): @@ -242,14 +247,33 @@ return ZstdCompressionParameters(**kwargs) - def __init__(self, format=0, compression_level=0, window_log=0, hash_log=0, - chain_log=0, search_log=0, min_match=0, target_length=0, - strategy=-1, compression_strategy=-1, - write_content_size=1, write_checksum=0, - write_dict_id=0, job_size=0, overlap_log=-1, - overlap_size_log=-1, force_max_window=0, enable_ldm=0, - ldm_hash_log=0, ldm_min_match=0, ldm_bucket_size_log=0, - ldm_hash_rate_log=-1, ldm_hash_every_log=-1, threads=0): + def __init__( + self, + format=0, + compression_level=0, + window_log=0, + hash_log=0, + chain_log=0, + search_log=0, + min_match=0, + target_length=0, + strategy=-1, + compression_strategy=-1, + write_content_size=1, + write_checksum=0, + write_dict_id=0, + job_size=0, + overlap_log=-1, + overlap_size_log=-1, + force_max_window=0, + enable_ldm=0, + ldm_hash_log=0, + ldm_min_match=0, + ldm_bucket_size_log=0, + ldm_hash_rate_log=-1, + ldm_hash_every_log=-1, + threads=0, + ): params = lib.ZSTD_createCCtxParams() if params == ffi.NULL: @@ -267,7 +291,9 @@ _set_compression_parameter(params, lib.ZSTD_c_nbWorkers, threads) _set_compression_parameter(params, lib.ZSTD_c_format, format) - _set_compression_parameter(params, lib.ZSTD_c_compressionLevel, compression_level) + _set_compression_parameter( + params, lib.ZSTD_c_compressionLevel, compression_level + ) _set_compression_parameter(params, lib.ZSTD_c_windowLog, window_log) _set_compression_parameter(params, lib.ZSTD_c_hashLog, hash_log) _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log) @@ -276,7 +302,7 @@ _set_compression_parameter(params, lib.ZSTD_c_targetLength, target_length) if strategy != -1 and compression_strategy != -1: - raise ValueError('cannot specify both compression_strategy and strategy') + raise ValueError("cannot specify both compression_strategy and strategy") if compression_strategy != -1: strategy = compression_strategy @@ -284,13 +310,15 @@ strategy = 0 _set_compression_parameter(params, lib.ZSTD_c_strategy, strategy) - _set_compression_parameter(params, lib.ZSTD_c_contentSizeFlag, write_content_size) + _set_compression_parameter( + params, lib.ZSTD_c_contentSizeFlag, write_content_size + ) _set_compression_parameter(params, lib.ZSTD_c_checksumFlag, write_checksum) _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id) _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size) if overlap_log != -1 and overlap_size_log != -1: - raise ValueError('cannot specify both overlap_log and overlap_size_log') + raise ValueError("cannot specify both overlap_log and overlap_size_log") if overlap_size_log != -1: overlap_log = overlap_size_log @@ -299,13 +327,19 @@ _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log) _set_compression_parameter(params, lib.ZSTD_c_forceMaxWindow, force_max_window) - _set_compression_parameter(params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm) + _set_compression_parameter( + params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm + ) _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log) _set_compression_parameter(params, lib.ZSTD_c_ldmMinMatch, ldm_min_match) - _set_compression_parameter(params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log) + _set_compression_parameter( + params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log + ) if ldm_hash_rate_log != -1 and ldm_hash_every_log != -1: - raise ValueError('cannot specify both ldm_hash_rate_log and ldm_hash_every_log') + raise ValueError( + "cannot specify both ldm_hash_rate_log and ldm_hash_every_log" + ) if ldm_hash_every_log != -1: ldm_hash_rate_log = ldm_hash_every_log @@ -380,7 +414,9 @@ @property def enable_ldm(self): - return _get_compression_parameter(self._params, lib.ZSTD_c_enableLongDistanceMatching) + return _get_compression_parameter( + self._params, lib.ZSTD_c_enableLongDistanceMatching + ) @property def ldm_hash_log(self): @@ -409,8 +445,10 @@ def estimated_compression_context_size(self): return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self._params) + CompressionParameters = ZstdCompressionParameters + def estimate_decompression_context_size(): return lib.ZSTD_estimateDCtxSize() @@ -418,24 +456,25 @@ def _set_compression_parameter(params, param, value): zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value) if lib.ZSTD_isError(zresult): - raise ZstdError('unable to set compression context parameter: %s' % - _zstd_error(zresult)) + raise ZstdError( + "unable to set compression context parameter: %s" % _zstd_error(zresult) + ) def _get_compression_parameter(params, param): - result = ffi.new('int *') + result = ffi.new("int *") zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result) if lib.ZSTD_isError(zresult): - raise ZstdError('unable to get compression context parameter: %s' % - _zstd_error(zresult)) + raise ZstdError( + "unable to get compression context parameter: %s" % _zstd_error(zresult) + ) return result[0] class ZstdCompressionWriter(object): - def __init__(self, compressor, writer, source_size, write_size, - write_return_read): + def __init__(self, compressor, writer, source_size, write_size, write_return_read): self._compressor = compressor self._writer = writer self._write_size = write_size @@ -444,24 +483,22 @@ self._closed = False self._bytes_compressed = 0 - self._dst_buffer = ffi.new('char[]', write_size) - self._out_buffer = ffi.new('ZSTD_outBuffer *') + self._dst_buffer = ffi.new("char[]", write_size) + self._out_buffer = ffi.new("ZSTD_outBuffer *") self._out_buffer.dst = self._dst_buffer self._out_buffer.size = len(self._dst_buffer) self._out_buffer.pos = 0 - zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, - source_size) + zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, source_size) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) def __enter__(self): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if self._entered: - raise ZstdError('cannot __enter__ multiple times') + raise ZstdError("cannot __enter__ multiple times") self._entered = True return self @@ -480,11 +517,11 @@ return lib.ZSTD_sizeof_CCtx(self._compressor._cctx) def fileno(self): - f = getattr(self._writer, 'fileno', None) + f = getattr(self._writer, "fileno", None) if f: return f() else: - raise OSError('fileno not available on underlying writer') + raise OSError("fileno not available on underlying writer") def close(self): if self._closed: @@ -496,7 +533,7 @@ self._closed = True # Call close() on underlying stream as well. - f = getattr(self._writer, 'close', None) + f = getattr(self._writer, "close", None) if f: f() @@ -529,7 +566,7 @@ return True def writelines(self, lines): - raise NotImplementedError('writelines() is not yet implemented') + raise NotImplementedError("writelines() is not yet implemented") def read(self, size=-1): raise io.UnsupportedOperation() @@ -542,13 +579,13 @@ def write(self, data): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") total_write = 0 data_buffer = ffi.from_buffer(data) - in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer = ffi.new("ZSTD_inBuffer *") in_buffer.src = data_buffer in_buffer.size = len(data_buffer) in_buffer.pos = 0 @@ -557,12 +594,11 @@ out_buffer.pos = 0 while in_buffer.pos < in_buffer.size: - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, in_buffer, - lib.ZSTD_e_continue) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if out_buffer.pos: self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:]) @@ -581,28 +617,27 @@ elif flush_mode == FLUSH_FRAME: flush = lib.ZSTD_e_end else: - raise ValueError('unknown flush_mode: %r' % flush_mode) + raise ValueError("unknown flush_mode: %r" % flush_mode) if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") total_write = 0 out_buffer = self._out_buffer out_buffer.pos = 0 - in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer = ffi.new("ZSTD_inBuffer *") in_buffer.src = ffi.NULL in_buffer.size = 0 in_buffer.pos = 0 while True: - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, in_buffer, - flush) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, in_buffer, flush + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if out_buffer.pos: self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:]) @@ -622,10 +657,10 @@ class ZstdCompressionObj(object): def compress(self, data): if self._finished: - raise ZstdError('cannot call compress() after compressor finished') + raise ZstdError("cannot call compress() after compressor finished") data_buffer = ffi.from_buffer(data) - source = ffi.new('ZSTD_inBuffer *') + source = ffi.new("ZSTD_inBuffer *") source.src = data_buffer source.size = len(data_buffer) source.pos = 0 @@ -633,26 +668,24 @@ chunks = [] while source.pos < len(data): - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - self._out, - source, - lib.ZSTD_e_continue) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, self._out, source, lib.ZSTD_e_continue + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if self._out.pos: chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:]) self._out.pos = 0 - return b''.join(chunks) + return b"".join(chunks) def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH): if flush_mode not in (COMPRESSOBJ_FLUSH_FINISH, COMPRESSOBJ_FLUSH_BLOCK): - raise ValueError('flush mode not recognized') + raise ValueError("flush mode not recognized") if self._finished: - raise ZstdError('compressor object already finished') + raise ZstdError("compressor object already finished") if flush_mode == COMPRESSOBJ_FLUSH_BLOCK: z_flush_mode = lib.ZSTD_e_flush @@ -660,11 +693,11 @@ z_flush_mode = lib.ZSTD_e_end self._finished = True else: - raise ZstdError('unhandled flush mode') + raise ZstdError("unhandled flush mode") assert self._out.pos == 0 - in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer = ffi.new("ZSTD_inBuffer *") in_buffer.src = ffi.NULL in_buffer.size = 0 in_buffer.pos = 0 @@ -672,13 +705,13 @@ chunks = [] while True: - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - self._out, - in_buffer, - z_flush_mode) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, self._out, in_buffer, z_flush_mode + ) if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s' % - _zstd_error(zresult)) + raise ZstdError( + "error ending compression stream: %s" % _zstd_error(zresult) + ) if self._out.pos: chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:]) @@ -687,19 +720,19 @@ if not zresult: break - return b''.join(chunks) + return b"".join(chunks) class ZstdCompressionChunker(object): def __init__(self, compressor, chunk_size): self._compressor = compressor - self._out = ffi.new('ZSTD_outBuffer *') - self._dst_buffer = ffi.new('char[]', chunk_size) + self._out = ffi.new("ZSTD_outBuffer *") + self._dst_buffer = ffi.new("char[]", chunk_size) self._out.dst = self._dst_buffer self._out.size = chunk_size self._out.pos = 0 - self._in = ffi.new('ZSTD_inBuffer *') + self._in = ffi.new("ZSTD_inBuffer *") self._in.src = ffi.NULL self._in.size = 0 self._in.pos = 0 @@ -707,11 +740,13 @@ def compress(self, data): if self._finished: - raise ZstdError('cannot call compress() after compression finished') + raise ZstdError("cannot call compress() after compression finished") if self._in.src != ffi.NULL: - raise ZstdError('cannot perform operation before consuming output ' - 'from previous operation') + raise ZstdError( + "cannot perform operation before consuming output " + "from previous operation" + ) data_buffer = ffi.from_buffer(data) @@ -723,10 +758,9 @@ self._in.pos = 0 while self._in.pos < self._in.size: - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - self._out, - self._in, - lib.ZSTD_e_continue) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, self._out, self._in, lib.ZSTD_e_continue + ) if self._in.pos == self._in.size: self._in.src = ffi.NULL @@ -734,8 +768,7 @@ self._in.pos = 0 if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if self._out.pos == self._out.size: yield ffi.buffer(self._out.dst, self._out.pos)[:] @@ -743,18 +776,19 @@ def flush(self): if self._finished: - raise ZstdError('cannot call flush() after compression finished') + raise ZstdError("cannot call flush() after compression finished") if self._in.src != ffi.NULL: - raise ZstdError('cannot call flush() before consuming output from ' - 'previous operation') + raise ZstdError( + "cannot call flush() before consuming output from " "previous operation" + ) while True: - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - self._out, self._in, - lib.ZSTD_e_flush) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, self._out, self._in, lib.ZSTD_e_flush + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if self._out.pos: yield ffi.buffer(self._out.dst, self._out.pos)[:] @@ -765,18 +799,20 @@ def finish(self): if self._finished: - raise ZstdError('cannot call finish() after compression finished') + raise ZstdError("cannot call finish() after compression finished") if self._in.src != ffi.NULL: - raise ZstdError('cannot call finish() before consuming output from ' - 'previous operation') + raise ZstdError( + "cannot call finish() before consuming output from " + "previous operation" + ) while True: - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - self._out, self._in, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, self._out, self._in, lib.ZSTD_e_end + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if self._out.pos: yield ffi.buffer(self._out.dst, self._out.pos)[:] @@ -798,13 +834,13 @@ self._finished_input = False self._finished_output = False - self._in_buffer = ffi.new('ZSTD_inBuffer *') + self._in_buffer = ffi.new("ZSTD_inBuffer *") # Holds a ref so backing bytes in self._in_buffer stay alive. self._source_buffer = None def __enter__(self): if self._entered: - raise ValueError('cannot __enter__ multiple times') + raise ValueError("cannot __enter__ multiple times") self._entered = True return self @@ -833,10 +869,10 @@ raise io.UnsupportedOperation() def write(self, data): - raise OSError('stream is not writable') + raise OSError("stream is not writable") def writelines(self, ignored): - raise OSError('stream is not writable') + raise OSError("stream is not writable") def isatty(self): return False @@ -865,7 +901,7 @@ chunks.append(chunk) - return b''.join(chunks) + return b"".join(chunks) def __iter__(self): raise io.UnsupportedOperation() @@ -879,7 +915,7 @@ if self._finished_input: return - if hasattr(self._source, 'read'): + if hasattr(self._source, "read"): data = self._source.read(self._read_size) if not data: @@ -902,9 +938,9 @@ old_pos = out_buffer.pos - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, self._in_buffer, - lib.ZSTD_e_continue) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_continue + ) self._bytes_compressed += out_buffer.pos - old_pos @@ -914,31 +950,30 @@ self._in_buffer.size = 0 self._source_buffer = None - if not hasattr(self._source, 'read'): + if not hasattr(self._source, "read"): self._finished_input = True if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s', - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s", _zstd_error(zresult)) return out_buffer.pos and out_buffer.pos == out_buffer.size def read(self, size=-1): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if size < -1: - raise ValueError('cannot read negative amounts less than -1') + raise ValueError("cannot read negative amounts less than -1") if size == -1: return self.readall() if self._finished_output or size == 0: - return b'' + return b"" # Need a dedicated ref to dest buffer otherwise it gets collected. - dst_buffer = ffi.new('char[]', size) - out_buffer = ffi.new('ZSTD_outBuffer *') + dst_buffer = ffi.new("char[]", size) + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dst_buffer out_buffer.size = size out_buffer.pos = 0 @@ -955,15 +990,14 @@ # EOF old_pos = out_buffer.pos - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, self._in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end + ) self._bytes_compressed += out_buffer.pos - old_pos if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s', - _zstd_error(zresult)) + raise ZstdError("error ending compression stream: %s", _zstd_error(zresult)) if zresult == 0: self._finished_output = True @@ -972,20 +1006,20 @@ def read1(self, size=-1): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if size < -1: - raise ValueError('cannot read negative amounts less than -1') + raise ValueError("cannot read negative amounts less than -1") if self._finished_output or size == 0: - return b'' + return b"" # -1 returns arbitrary number of bytes. if size == -1: size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE - dst_buffer = ffi.new('char[]', size) - out_buffer = ffi.new('ZSTD_outBuffer *') + dst_buffer = ffi.new("char[]", size) + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dst_buffer out_buffer.size = size out_buffer.pos = 0 @@ -1020,15 +1054,16 @@ # EOF. old_pos = out_buffer.pos - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, self._in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end + ) self._bytes_compressed += out_buffer.pos - old_pos if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s' % - _zstd_error(zresult)) + raise ZstdError( + "error ending compression stream: %s" % _zstd_error(zresult) + ) if zresult == 0: self._finished_output = True @@ -1037,15 +1072,15 @@ def readinto(self, b): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if self._finished_output: return 0 # TODO use writable=True once we require CFFI >= 1.12. dest_buffer = ffi.from_buffer(b) - ffi.memmove(b, b'', 0) - out_buffer = ffi.new('ZSTD_outBuffer *') + ffi.memmove(b, b"", 0) + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dest_buffer out_buffer.size = len(dest_buffer) out_buffer.pos = 0 @@ -1060,15 +1095,14 @@ # EOF. old_pos = out_buffer.pos - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, self._in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end + ) self._bytes_compressed += out_buffer.pos - old_pos if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s', - _zstd_error(zresult)) + raise ZstdError("error ending compression stream: %s", _zstd_error(zresult)) if zresult == 0: self._finished_output = True @@ -1077,16 +1111,16 @@ def readinto1(self, b): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if self._finished_output: return 0 # TODO use writable=True once we require CFFI >= 1.12. dest_buffer = ffi.from_buffer(b) - ffi.memmove(b, b'', 0) - - out_buffer = ffi.new('ZSTD_outBuffer *') + ffi.memmove(b, b"", 0) + + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dest_buffer out_buffer.size = len(dest_buffer) out_buffer.pos = 0 @@ -1107,15 +1141,16 @@ # EOF. old_pos = out_buffer.pos - zresult = lib.ZSTD_compressStream2(self._compressor._cctx, - out_buffer, self._in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end + ) self._bytes_compressed += out_buffer.pos - old_pos if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s' % - _zstd_error(zresult)) + raise ZstdError( + "error ending compression stream: %s" % _zstd_error(zresult) + ) if zresult == 0: self._finished_output = True @@ -1124,29 +1159,35 @@ class ZstdCompressor(object): - def __init__(self, level=3, dict_data=None, compression_params=None, - write_checksum=None, write_content_size=None, - write_dict_id=None, threads=0): + def __init__( + self, + level=3, + dict_data=None, + compression_params=None, + write_checksum=None, + write_content_size=None, + write_dict_id=None, + threads=0, + ): if level > lib.ZSTD_maxCLevel(): - raise ValueError('level must be less than %d' % lib.ZSTD_maxCLevel()) + raise ValueError("level must be less than %d" % lib.ZSTD_maxCLevel()) if threads < 0: threads = _cpu_count() if compression_params and write_checksum is not None: - raise ValueError('cannot define compression_params and ' - 'write_checksum') + raise ValueError("cannot define compression_params and " "write_checksum") if compression_params and write_content_size is not None: - raise ValueError('cannot define compression_params and ' - 'write_content_size') + raise ValueError( + "cannot define compression_params and " "write_content_size" + ) if compression_params and write_dict_id is not None: - raise ValueError('cannot define compression_params and ' - 'write_dict_id') + raise ValueError("cannot define compression_params and " "write_dict_id") if compression_params and threads: - raise ValueError('cannot define compression_params and threads') + raise ValueError("cannot define compression_params and threads") if compression_params: self._params = _make_cctx_params(compression_params) @@ -1160,27 +1201,24 @@ self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams) - _set_compression_parameter(self._params, - lib.ZSTD_c_compressionLevel, - level) + _set_compression_parameter(self._params, lib.ZSTD_c_compressionLevel, level) _set_compression_parameter( self._params, lib.ZSTD_c_contentSizeFlag, - write_content_size if write_content_size is not None else 1) - - _set_compression_parameter(self._params, - lib.ZSTD_c_checksumFlag, - 1 if write_checksum else 0) - - _set_compression_parameter(self._params, - lib.ZSTD_c_dictIDFlag, - 1 if write_dict_id else 0) + write_content_size if write_content_size is not None else 1, + ) + + _set_compression_parameter( + self._params, lib.ZSTD_c_checksumFlag, 1 if write_checksum else 0 + ) + + _set_compression_parameter( + self._params, lib.ZSTD_c_dictIDFlag, 1 if write_dict_id else 0 + ) if threads: - _set_compression_parameter(self._params, - lib.ZSTD_c_nbWorkers, - threads) + _set_compression_parameter(self._params, lib.ZSTD_c_nbWorkers, threads) cctx = lib.ZSTD_createCCtx() if cctx == ffi.NULL: @@ -1194,15 +1232,16 @@ try: self._setup_cctx() finally: - self._cctx = ffi.gc(cctx, lib.ZSTD_freeCCtx, - size=lib.ZSTD_sizeof_CCtx(cctx)) + self._cctx = ffi.gc( + cctx, lib.ZSTD_freeCCtx, size=lib.ZSTD_sizeof_CCtx(cctx) + ) def _setup_cctx(self): - zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(self._cctx, - self._params) + zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(self._cctx, self._params) if lib.ZSTD_isError(zresult): - raise ZstdError('could not set compression parameters: %s' % - _zstd_error(zresult)) + raise ZstdError( + "could not set compression parameters: %s" % _zstd_error(zresult) + ) dict_data = self._dict_data @@ -1211,12 +1250,17 @@ zresult = lib.ZSTD_CCtx_refCDict(self._cctx, dict_data._cdict) else: zresult = lib.ZSTD_CCtx_loadDictionary_advanced( - self._cctx, dict_data.as_bytes(), len(dict_data), - lib.ZSTD_dlm_byRef, dict_data._dict_type) + self._cctx, + dict_data.as_bytes(), + len(dict_data), + lib.ZSTD_dlm_byRef, + dict_data._dict_type, + ) if lib.ZSTD_isError(zresult): - raise ZstdError('could not load compression dictionary: %s' % - _zstd_error(zresult)) + raise ZstdError( + "could not load compression dictionary: %s" % _zstd_error(zresult) + ) def memory_size(self): return lib.ZSTD_sizeof_CCtx(self._cctx) @@ -1227,15 +1271,14 @@ data_buffer = ffi.from_buffer(data) dest_size = lib.ZSTD_compressBound(len(data_buffer)) - out = new_nonzero('char[]', dest_size) + out = new_nonzero("char[]", dest_size) zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer)) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) - - out_buffer = ffi.new('ZSTD_outBuffer *') - in_buffer = ffi.new('ZSTD_inBuffer *') + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) + + out_buffer = ffi.new("ZSTD_outBuffer *") + in_buffer = ffi.new("ZSTD_inBuffer *") out_buffer.dst = out out_buffer.size = dest_size @@ -1245,16 +1288,14 @@ in_buffer.size = len(data_buffer) in_buffer.pos = 0 - zresult = lib.ZSTD_compressStream2(self._cctx, - out_buffer, - in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end + ) if lib.ZSTD_isError(zresult): - raise ZstdError('cannot compress: %s' % - _zstd_error(zresult)) + raise ZstdError("cannot compress: %s" % _zstd_error(zresult)) elif zresult: - raise ZstdError('unexpected partial frame flush') + raise ZstdError("unexpected partial frame flush") return ffi.buffer(out, out_buffer.pos)[:] @@ -1266,12 +1307,11 @@ zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) cobj = ZstdCompressionObj() - cobj._out = ffi.new('ZSTD_outBuffer *') - cobj._dst_buffer = ffi.new('char[]', COMPRESSION_RECOMMENDED_OUTPUT_SIZE) + cobj._out = ffi.new("ZSTD_outBuffer *") + cobj._dst_buffer = ffi.new("char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE) cobj._out.dst = cobj._dst_buffer cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE cobj._out.pos = 0 @@ -1288,19 +1328,23 @@ zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) return ZstdCompressionChunker(self, chunk_size=chunk_size) - def copy_stream(self, ifh, ofh, size=-1, - read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE, - write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE): - - if not hasattr(ifh, 'read'): - raise ValueError('first argument must have a read() method') - if not hasattr(ofh, 'write'): - raise ValueError('second argument must have a write() method') + def copy_stream( + self, + ifh, + ofh, + size=-1, + read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE, + write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, + ): + + if not hasattr(ifh, "read"): + raise ValueError("first argument must have a read() method") + if not hasattr(ofh, "write"): + raise ValueError("second argument must have a write() method") lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) @@ -1309,13 +1353,12 @@ zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) - - in_buffer = ffi.new('ZSTD_inBuffer *') - out_buffer = ffi.new('ZSTD_outBuffer *') - - dst_buffer = ffi.new('char[]', write_size) + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) + + in_buffer = ffi.new("ZSTD_inBuffer *") + out_buffer = ffi.new("ZSTD_outBuffer *") + + dst_buffer = ffi.new("char[]", write_size) out_buffer.dst = dst_buffer out_buffer.size = write_size out_buffer.pos = 0 @@ -1334,13 +1377,11 @@ in_buffer.pos = 0 while in_buffer.pos < in_buffer.size: - zresult = lib.ZSTD_compressStream2(self._cctx, - out_buffer, - in_buffer, - lib.ZSTD_e_continue) + zresult = lib.ZSTD_compressStream2( + self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if out_buffer.pos: ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) @@ -1349,13 +1390,13 @@ # We've finished reading. Flush the compressor. while True: - zresult = lib.ZSTD_compressStream2(self._cctx, - out_buffer, - in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end + ) if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s' % - _zstd_error(zresult)) + raise ZstdError( + "error ending compression stream: %s" % _zstd_error(zresult) + ) if out_buffer.pos: ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) @@ -1367,8 +1408,9 @@ return total_read, total_write - def stream_reader(self, source, size=-1, - read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE): + def stream_reader( + self, source, size=-1, read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE + ): lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) try: @@ -1381,40 +1423,48 @@ zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) return ZstdCompressionReader(self, source, read_size) - def stream_writer(self, writer, size=-1, - write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, - write_return_read=False): - - if not hasattr(writer, 'write'): - raise ValueError('must pass an object with a write() method') + def stream_writer( + self, + writer, + size=-1, + write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, + write_return_read=False, + ): + + if not hasattr(writer, "write"): + raise ValueError("must pass an object with a write() method") lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) if size < 0: size = lib.ZSTD_CONTENTSIZE_UNKNOWN - return ZstdCompressionWriter(self, writer, size, write_size, - write_return_read) + return ZstdCompressionWriter(self, writer, size, write_size, write_return_read) write_to = stream_writer - def read_to_iter(self, reader, size=-1, - read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE, - write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE): - if hasattr(reader, 'read'): + def read_to_iter( + self, + reader, + size=-1, + read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE, + write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, + ): + if hasattr(reader, "read"): have_read = True - elif hasattr(reader, '__getitem__'): + elif hasattr(reader, "__getitem__"): have_read = False buffer_offset = 0 size = len(reader) else: - raise ValueError('must pass an object with a read() method or ' - 'conforms to buffer protocol') + raise ValueError( + "must pass an object with a read() method or " + "conforms to buffer protocol" + ) lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) @@ -1423,17 +1473,16 @@ zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) if lib.ZSTD_isError(zresult): - raise ZstdError('error setting source size: %s' % - _zstd_error(zresult)) - - in_buffer = ffi.new('ZSTD_inBuffer *') - out_buffer = ffi.new('ZSTD_outBuffer *') + raise ZstdError("error setting source size: %s" % _zstd_error(zresult)) + + in_buffer = ffi.new("ZSTD_inBuffer *") + out_buffer = ffi.new("ZSTD_outBuffer *") in_buffer.src = ffi.NULL in_buffer.size = 0 in_buffer.pos = 0 - dst_buffer = ffi.new('char[]', write_size) + dst_buffer = ffi.new("char[]", write_size) out_buffer.dst = dst_buffer out_buffer.size = write_size out_buffer.pos = 0 @@ -1449,7 +1498,7 @@ else: remaining = len(reader) - buffer_offset slice_size = min(remaining, read_size) - read_result = reader[buffer_offset:buffer_offset + slice_size] + read_result = reader[buffer_offset : buffer_offset + slice_size] buffer_offset += slice_size # No new input data. Break out of the read loop. @@ -1464,11 +1513,11 @@ in_buffer.pos = 0 while in_buffer.pos < in_buffer.size: - zresult = lib.ZSTD_compressStream2(self._cctx, out_buffer, in_buffer, - lib.ZSTD_e_continue) + zresult = lib.ZSTD_compressStream2( + self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd compress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd compress error: %s" % _zstd_error(zresult)) if out_buffer.pos: data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:] @@ -1484,13 +1533,13 @@ # remains. while True: assert out_buffer.pos == 0 - zresult = lib.ZSTD_compressStream2(self._cctx, - out_buffer, - in_buffer, - lib.ZSTD_e_end) + zresult = lib.ZSTD_compressStream2( + self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end + ) if lib.ZSTD_isError(zresult): - raise ZstdError('error ending compression stream: %s' % - _zstd_error(zresult)) + raise ZstdError( + "error ending compression stream: %s" % _zstd_error(zresult) + ) if out_buffer.pos: data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:] @@ -1522,7 +1571,7 @@ size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer)) if size == lib.ZSTD_CONTENTSIZE_ERROR: - raise ZstdError('error when determining content size') + raise ZstdError("error when determining content size") elif size == lib.ZSTD_CONTENTSIZE_UNKNOWN: return -1 else: @@ -1534,24 +1583,23 @@ zresult = lib.ZSTD_frameHeaderSize(data_buffer, len(data_buffer)) if lib.ZSTD_isError(zresult): - raise ZstdError('could not determine frame header size: %s' % - _zstd_error(zresult)) + raise ZstdError( + "could not determine frame header size: %s" % _zstd_error(zresult) + ) return zresult def get_frame_parameters(data): - params = ffi.new('ZSTD_frameHeader *') + params = ffi.new("ZSTD_frameHeader *") data_buffer = ffi.from_buffer(data) zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer)) if lib.ZSTD_isError(zresult): - raise ZstdError('cannot get frame parameters: %s' % - _zstd_error(zresult)) + raise ZstdError("cannot get frame parameters: %s" % _zstd_error(zresult)) if zresult: - raise ZstdError('not enough data for frame parameters; need %d bytes' % - zresult) + raise ZstdError("not enough data for frame parameters; need %d bytes" % zresult) return FrameParameters(params[0]) @@ -1563,10 +1611,10 @@ self.k = k self.d = d - if dict_type not in (DICT_TYPE_AUTO, DICT_TYPE_RAWCONTENT, - DICT_TYPE_FULLDICT): - raise ValueError('invalid dictionary load mode: %d; must use ' - 'DICT_TYPE_* constants') + if dict_type not in (DICT_TYPE_AUTO, DICT_TYPE_RAWCONTENT, DICT_TYPE_FULLDICT): + raise ValueError( + "invalid dictionary load mode: %d; must use " "DICT_TYPE_* constants" + ) self._dict_type = dict_type self._cdict = None @@ -1582,16 +1630,15 @@ def precompute_compress(self, level=0, compression_params=None): if level and compression_params: - raise ValueError('must only specify one of level or ' - 'compression_params') + raise ValueError("must only specify one of level or " "compression_params") if not level and not compression_params: - raise ValueError('must specify one of level or compression_params') + raise ValueError("must specify one of level or compression_params") if level: cparams = lib.ZSTD_getCParams(level, 0, len(self._data)) else: - cparams = ffi.new('ZSTD_compressionParameters') + cparams = ffi.new("ZSTD_compressionParameters") cparams.chainLog = compression_params.chain_log cparams.hashLog = compression_params.hash_log cparams.minMatch = compression_params.min_match @@ -1600,59 +1647,75 @@ cparams.targetLength = compression_params.target_length cparams.windowLog = compression_params.window_log - cdict = lib.ZSTD_createCDict_advanced(self._data, len(self._data), - lib.ZSTD_dlm_byRef, - self._dict_type, - cparams, - lib.ZSTD_defaultCMem) + cdict = lib.ZSTD_createCDict_advanced( + self._data, + len(self._data), + lib.ZSTD_dlm_byRef, + self._dict_type, + cparams, + lib.ZSTD_defaultCMem, + ) if cdict == ffi.NULL: - raise ZstdError('unable to precompute dictionary') - - self._cdict = ffi.gc(cdict, lib.ZSTD_freeCDict, - size=lib.ZSTD_sizeof_CDict(cdict)) + raise ZstdError("unable to precompute dictionary") + + self._cdict = ffi.gc( + cdict, lib.ZSTD_freeCDict, size=lib.ZSTD_sizeof_CDict(cdict) + ) @property def _ddict(self): - ddict = lib.ZSTD_createDDict_advanced(self._data, len(self._data), - lib.ZSTD_dlm_byRef, - self._dict_type, - lib.ZSTD_defaultCMem) + ddict = lib.ZSTD_createDDict_advanced( + self._data, + len(self._data), + lib.ZSTD_dlm_byRef, + self._dict_type, + lib.ZSTD_defaultCMem, + ) if ddict == ffi.NULL: - raise ZstdError('could not create decompression dict') - - ddict = ffi.gc(ddict, lib.ZSTD_freeDDict, - size=lib.ZSTD_sizeof_DDict(ddict)) - self.__dict__['_ddict'] = ddict + raise ZstdError("could not create decompression dict") + + ddict = ffi.gc(ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict)) + self.__dict__["_ddict"] = ddict return ddict -def train_dictionary(dict_size, samples, k=0, d=0, notifications=0, dict_id=0, - level=0, steps=0, threads=0): + +def train_dictionary( + dict_size, + samples, + k=0, + d=0, + notifications=0, + dict_id=0, + level=0, + steps=0, + threads=0, +): if not isinstance(samples, list): - raise TypeError('samples must be a list') + raise TypeError("samples must be a list") if threads < 0: threads = _cpu_count() total_size = sum(map(len, samples)) - samples_buffer = new_nonzero('char[]', total_size) - sample_sizes = new_nonzero('size_t[]', len(samples)) + samples_buffer = new_nonzero("char[]", total_size) + sample_sizes = new_nonzero("size_t[]", len(samples)) offset = 0 for i, sample in enumerate(samples): if not isinstance(sample, bytes_type): - raise ValueError('samples must be bytes') + raise ValueError("samples must be bytes") l = len(sample) ffi.memmove(samples_buffer + offset, sample, l) offset += l sample_sizes[i] = l - dict_data = new_nonzero('char[]', dict_size) - - dparams = ffi.new('ZDICT_cover_params_t *')[0] + dict_data = new_nonzero("char[]", dict_size) + + dparams = ffi.new("ZDICT_cover_params_t *")[0] dparams.k = k dparams.d = d dparams.steps = steps @@ -1661,34 +1724,51 @@ dparams.zParams.dictID = dict_id dparams.zParams.compressionLevel = level - if (not dparams.k and not dparams.d and not dparams.steps - and not dparams.nbThreads and not dparams.zParams.notificationLevel + if ( + not dparams.k + and not dparams.d + and not dparams.steps + and not dparams.nbThreads + and not dparams.zParams.notificationLevel and not dparams.zParams.dictID - and not dparams.zParams.compressionLevel): + and not dparams.zParams.compressionLevel + ): zresult = lib.ZDICT_trainFromBuffer( - ffi.addressof(dict_data), dict_size, + ffi.addressof(dict_data), + dict_size, ffi.addressof(samples_buffer), - ffi.addressof(sample_sizes, 0), len(samples)) + ffi.addressof(sample_sizes, 0), + len(samples), + ) elif dparams.steps or dparams.nbThreads: zresult = lib.ZDICT_optimizeTrainFromBuffer_cover( - ffi.addressof(dict_data), dict_size, + ffi.addressof(dict_data), + dict_size, ffi.addressof(samples_buffer), - ffi.addressof(sample_sizes, 0), len(samples), - ffi.addressof(dparams)) + ffi.addressof(sample_sizes, 0), + len(samples), + ffi.addressof(dparams), + ) else: zresult = lib.ZDICT_trainFromBuffer_cover( - ffi.addressof(dict_data), dict_size, + ffi.addressof(dict_data), + dict_size, ffi.addressof(samples_buffer), - ffi.addressof(sample_sizes, 0), len(samples), - dparams) + ffi.addressof(sample_sizes, 0), + len(samples), + dparams, + ) if lib.ZDICT_isError(zresult): - msg = ffi.string(lib.ZDICT_getErrorName(zresult)).decode('utf-8') - raise ZstdError('cannot train dict: %s' % msg) - - return ZstdCompressionDict(ffi.buffer(dict_data, zresult)[:], - dict_type=DICT_TYPE_FULLDICT, - k=dparams.k, d=dparams.d) + msg = ffi.string(lib.ZDICT_getErrorName(zresult)).decode("utf-8") + raise ZstdError("cannot train dict: %s" % msg) + + return ZstdCompressionDict( + ffi.buffer(dict_data, zresult)[:], + dict_type=DICT_TYPE_FULLDICT, + k=dparams.k, + d=dparams.d, + ) class ZstdDecompressionObj(object): @@ -1699,21 +1779,21 @@ def decompress(self, data): if self._finished: - raise ZstdError('cannot use a decompressobj multiple times') - - in_buffer = ffi.new('ZSTD_inBuffer *') - out_buffer = ffi.new('ZSTD_outBuffer *') + raise ZstdError("cannot use a decompressobj multiple times") + + in_buffer = ffi.new("ZSTD_inBuffer *") + out_buffer = ffi.new("ZSTD_outBuffer *") data_buffer = ffi.from_buffer(data) if len(data_buffer) == 0: - return b'' + return b"" in_buffer.src = data_buffer in_buffer.size = len(data_buffer) in_buffer.pos = 0 - dst_buffer = ffi.new('char[]', self._write_size) + dst_buffer = ffi.new("char[]", self._write_size) out_buffer.dst = dst_buffer out_buffer.size = len(dst_buffer) out_buffer.pos = 0 @@ -1721,11 +1801,11 @@ chunks = [] while True: - zresult = lib.ZSTD_decompressStream(self._decompressor._dctx, - out_buffer, in_buffer) + zresult = lib.ZSTD_decompressStream( + self._decompressor._dctx, out_buffer, in_buffer + ) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd decompressor error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd decompressor error: %s" % _zstd_error(zresult)) if zresult == 0: self._finished = True @@ -1734,13 +1814,14 @@ if out_buffer.pos: chunks.append(ffi.buffer(out_buffer.dst, out_buffer.pos)[:]) - if (zresult == 0 or - (in_buffer.pos == in_buffer.size and out_buffer.pos == 0)): + if zresult == 0 or ( + in_buffer.pos == in_buffer.size and out_buffer.pos == 0 + ): break out_buffer.pos = 0 - return b''.join(chunks) + return b"".join(chunks) def flush(self, length=0): pass @@ -1757,13 +1838,13 @@ self._bytes_decompressed = 0 self._finished_input = False self._finished_output = False - self._in_buffer = ffi.new('ZSTD_inBuffer *') + self._in_buffer = ffi.new("ZSTD_inBuffer *") # Holds a ref to self._in_buffer.src. self._source_buffer = None def __enter__(self): if self._entered: - raise ValueError('cannot __enter__ multiple times') + raise ValueError("cannot __enter__ multiple times") self._entered = True return self @@ -1824,7 +1905,7 @@ chunks.append(chunk) - return b''.join(chunks) + return b"".join(chunks) def __iter__(self): raise io.UnsupportedOperation() @@ -1844,7 +1925,7 @@ return # Else populate the input buffer from our source. - if hasattr(self._source, 'read'): + if hasattr(self._source, "read"): data = self._source.read(self._read_size) if not data: @@ -1866,8 +1947,9 @@ Returns True if data in output buffer should be emitted. """ - zresult = lib.ZSTD_decompressStream(self._decompressor._dctx, - out_buffer, self._in_buffer) + zresult = lib.ZSTD_decompressStream( + self._decompressor._dctx, out_buffer, self._in_buffer + ) if self._in_buffer.pos == self._in_buffer.size: self._in_buffer.src = ffi.NULL @@ -1875,38 +1957,39 @@ self._in_buffer.size = 0 self._source_buffer = None - if not hasattr(self._source, 'read'): + if not hasattr(self._source, "read"): self._finished_input = True if lib.ZSTD_isError(zresult): - raise ZstdError('zstd decompress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult)) # Emit data if there is data AND either: # a) output buffer is full (read amount is satisfied) # b) we're at end of a frame and not in frame spanning mode - return (out_buffer.pos and - (out_buffer.pos == out_buffer.size or - zresult == 0 and not self._read_across_frames)) + return out_buffer.pos and ( + out_buffer.pos == out_buffer.size + or zresult == 0 + and not self._read_across_frames + ) def read(self, size=-1): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if size < -1: - raise ValueError('cannot read negative amounts less than -1') + raise ValueError("cannot read negative amounts less than -1") if size == -1: # This is recursive. But it gets the job done. return self.readall() if self._finished_output or size == 0: - return b'' + return b"" # We /could/ call into readinto() here. But that introduces more # overhead. - dst_buffer = ffi.new('char[]', size) - out_buffer = ffi.new('ZSTD_outBuffer *') + dst_buffer = ffi.new("char[]", size) + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dst_buffer out_buffer.size = size out_buffer.pos = 0 @@ -1927,15 +2010,15 @@ def readinto(self, b): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if self._finished_output: return 0 # TODO use writable=True once we require CFFI >= 1.12. dest_buffer = ffi.from_buffer(b) - ffi.memmove(b, b'', 0) - out_buffer = ffi.new('ZSTD_outBuffer *') + ffi.memmove(b, b"", 0) + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dest_buffer out_buffer.size = len(dest_buffer) out_buffer.pos = 0 @@ -1956,20 +2039,20 @@ def read1(self, size=-1): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if size < -1: - raise ValueError('cannot read negative amounts less than -1') + raise ValueError("cannot read negative amounts less than -1") if self._finished_output or size == 0: - return b'' + return b"" # -1 returns arbitrary number of bytes. if size == -1: size = DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE - dst_buffer = ffi.new('char[]', size) - out_buffer = ffi.new('ZSTD_outBuffer *') + dst_buffer = ffi.new("char[]", size) + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dst_buffer out_buffer.size = size out_buffer.pos = 0 @@ -1990,16 +2073,16 @@ def readinto1(self, b): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if self._finished_output: return 0 # TODO use writable=True once we require CFFI >= 1.12. dest_buffer = ffi.from_buffer(b) - ffi.memmove(b, b'', 0) - - out_buffer = ffi.new('ZSTD_outBuffer *') + ffi.memmove(b, b"", 0) + + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = dest_buffer out_buffer.size = len(dest_buffer) out_buffer.pos = 0 @@ -2016,33 +2099,31 @@ def seek(self, pos, whence=os.SEEK_SET): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") read_amount = 0 if whence == os.SEEK_SET: if pos < 0: - raise ValueError('cannot seek to negative position with SEEK_SET') + raise ValueError("cannot seek to negative position with SEEK_SET") if pos < self._bytes_decompressed: - raise ValueError('cannot seek zstd decompression stream ' - 'backwards') + raise ValueError("cannot seek zstd decompression stream " "backwards") read_amount = pos - self._bytes_decompressed elif whence == os.SEEK_CUR: if pos < 0: - raise ValueError('cannot seek zstd decompression stream ' - 'backwards') + raise ValueError("cannot seek zstd decompression stream " "backwards") read_amount = pos elif whence == os.SEEK_END: - raise ValueError('zstd decompression streams cannot be seeked ' - 'with SEEK_END') + raise ValueError( + "zstd decompression streams cannot be seeked " "with SEEK_END" + ) while read_amount: - result = self.read(min(read_amount, - DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)) + result = self.read(min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)) if not result: break @@ -2051,6 +2132,7 @@ return self._bytes_decompressed + class ZstdDecompressionWriter(object): def __init__(self, decompressor, writer, write_size, write_return_read): decompressor._ensure_dctx() @@ -2064,10 +2146,10 @@ def __enter__(self): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") if self._entered: - raise ZstdError('cannot __enter__ multiple times') + raise ZstdError("cannot __enter__ multiple times") self._entered = True @@ -2089,7 +2171,7 @@ finally: self._closed = True - f = getattr(self._writer, 'close', None) + f = getattr(self._writer, "close", None) if f: f() @@ -2098,17 +2180,17 @@ return self._closed def fileno(self): - f = getattr(self._writer, 'fileno', None) + f = getattr(self._writer, "fileno", None) if f: return f() else: - raise OSError('fileno not available on underlying writer') + raise OSError("fileno not available on underlying writer") def flush(self): if self._closed: - raise ValueError('stream is closed') - - f = getattr(self._writer, 'flush', None) + raise ValueError("stream is closed") + + f = getattr(self._writer, "flush", None) if f: return f() @@ -2153,19 +2235,19 @@ def write(self, data): if self._closed: - raise ValueError('stream is closed') + raise ValueError("stream is closed") total_write = 0 - in_buffer = ffi.new('ZSTD_inBuffer *') - out_buffer = ffi.new('ZSTD_outBuffer *') + in_buffer = ffi.new("ZSTD_inBuffer *") + out_buffer = ffi.new("ZSTD_outBuffer *") data_buffer = ffi.from_buffer(data) in_buffer.src = data_buffer in_buffer.size = len(data_buffer) in_buffer.pos = 0 - dst_buffer = ffi.new('char[]', self._write_size) + dst_buffer = ffi.new("char[]", self._write_size) out_buffer.dst = dst_buffer out_buffer.size = len(dst_buffer) out_buffer.pos = 0 @@ -2175,8 +2257,7 @@ while in_buffer.pos < in_buffer.size: zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd decompress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult)) if out_buffer.pos: self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:]) @@ -2206,8 +2287,9 @@ try: self._ensure_dctx() finally: - self._dctx = ffi.gc(dctx, lib.ZSTD_freeDCtx, - size=lib.ZSTD_sizeof_DCtx(dctx)) + self._dctx = ffi.gc( + dctx, lib.ZSTD_freeDCtx, size=lib.ZSTD_sizeof_DCtx(dctx) + ) def memory_size(self): return lib.ZSTD_sizeof_DCtx(self._dctx) @@ -2220,85 +2302,96 @@ output_size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer)) if output_size == lib.ZSTD_CONTENTSIZE_ERROR: - raise ZstdError('error determining content size from frame header') + raise ZstdError("error determining content size from frame header") elif output_size == 0: - return b'' + return b"" elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN: if not max_output_size: - raise ZstdError('could not determine content size in frame header') - - result_buffer = ffi.new('char[]', max_output_size) + raise ZstdError("could not determine content size in frame header") + + result_buffer = ffi.new("char[]", max_output_size) result_size = max_output_size output_size = 0 else: - result_buffer = ffi.new('char[]', output_size) + result_buffer = ffi.new("char[]", output_size) result_size = output_size - out_buffer = ffi.new('ZSTD_outBuffer *') + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = result_buffer out_buffer.size = result_size out_buffer.pos = 0 - in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer = ffi.new("ZSTD_inBuffer *") in_buffer.src = data_buffer in_buffer.size = len(data_buffer) in_buffer.pos = 0 zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) if lib.ZSTD_isError(zresult): - raise ZstdError('decompression error: %s' % - _zstd_error(zresult)) + raise ZstdError("decompression error: %s" % _zstd_error(zresult)) elif zresult: - raise ZstdError('decompression error: did not decompress full frame') + raise ZstdError("decompression error: did not decompress full frame") elif output_size and out_buffer.pos != output_size: - raise ZstdError('decompression error: decompressed %d bytes; expected %d' % - (zresult, output_size)) + raise ZstdError( + "decompression error: decompressed %d bytes; expected %d" + % (zresult, output_size) + ) return ffi.buffer(result_buffer, out_buffer.pos)[:] - def stream_reader(self, source, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, - read_across_frames=False): + def stream_reader( + self, + source, + read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, + read_across_frames=False, + ): self._ensure_dctx() return ZstdDecompressionReader(self, source, read_size, read_across_frames) def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE): if write_size < 1: - raise ValueError('write_size must be positive') + raise ValueError("write_size must be positive") self._ensure_dctx() return ZstdDecompressionObj(self, write_size=write_size) - def read_to_iter(self, reader, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, - write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, - skip_bytes=0): + def read_to_iter( + self, + reader, + read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, + write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, + skip_bytes=0, + ): if skip_bytes >= read_size: - raise ValueError('skip_bytes must be smaller than read_size') - - if hasattr(reader, 'read'): + raise ValueError("skip_bytes must be smaller than read_size") + + if hasattr(reader, "read"): have_read = True - elif hasattr(reader, '__getitem__'): + elif hasattr(reader, "__getitem__"): have_read = False buffer_offset = 0 size = len(reader) else: - raise ValueError('must pass an object with a read() method or ' - 'conforms to buffer protocol') + raise ValueError( + "must pass an object with a read() method or " + "conforms to buffer protocol" + ) if skip_bytes: if have_read: reader.read(skip_bytes) else: if skip_bytes > size: - raise ValueError('skip_bytes larger than first input chunk') + raise ValueError("skip_bytes larger than first input chunk") buffer_offset = skip_bytes self._ensure_dctx() - in_buffer = ffi.new('ZSTD_inBuffer *') - out_buffer = ffi.new('ZSTD_outBuffer *') - - dst_buffer = ffi.new('char[]', write_size) + in_buffer = ffi.new("ZSTD_inBuffer *") + out_buffer = ffi.new("ZSTD_outBuffer *") + + dst_buffer = ffi.new("char[]", write_size) out_buffer.dst = dst_buffer out_buffer.size = len(dst_buffer) out_buffer.pos = 0 @@ -2311,7 +2404,7 @@ else: remaining = size - buffer_offset slice_size = min(remaining, read_size) - read_result = reader[buffer_offset:buffer_offset + slice_size] + read_result = reader[buffer_offset : buffer_offset + slice_size] buffer_offset += slice_size # No new input. Break out of read loop. @@ -2330,8 +2423,7 @@ zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd decompress error: %s' % - _zstd_error(zresult)) + raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult)) if out_buffer.pos: data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:] @@ -2348,30 +2440,37 @@ read_from = read_to_iter - def stream_writer(self, writer, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, - write_return_read=False): - if not hasattr(writer, 'write'): - raise ValueError('must pass an object with a write() method') - - return ZstdDecompressionWriter(self, writer, write_size, - write_return_read) + def stream_writer( + self, + writer, + write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, + write_return_read=False, + ): + if not hasattr(writer, "write"): + raise ValueError("must pass an object with a write() method") + + return ZstdDecompressionWriter(self, writer, write_size, write_return_read) write_to = stream_writer - def copy_stream(self, ifh, ofh, - read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, - write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE): - if not hasattr(ifh, 'read'): - raise ValueError('first argument must have a read() method') - if not hasattr(ofh, 'write'): - raise ValueError('second argument must have a write() method') + def copy_stream( + self, + ifh, + ofh, + read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, + write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, + ): + if not hasattr(ifh, "read"): + raise ValueError("first argument must have a read() method") + if not hasattr(ofh, "write"): + raise ValueError("second argument must have a write() method") self._ensure_dctx() - in_buffer = ffi.new('ZSTD_inBuffer *') - out_buffer = ffi.new('ZSTD_outBuffer *') - - dst_buffer = ffi.new('char[]', write_size) + in_buffer = ffi.new("ZSTD_inBuffer *") + out_buffer = ffi.new("ZSTD_outBuffer *") + + dst_buffer = ffi.new("char[]", write_size) out_buffer.dst = dst_buffer out_buffer.size = write_size out_buffer.pos = 0 @@ -2394,8 +2493,9 @@ while in_buffer.pos < in_buffer.size: zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) if lib.ZSTD_isError(zresult): - raise ZstdError('zstd decompressor error: %s' % - _zstd_error(zresult)) + raise ZstdError( + "zstd decompressor error: %s" % _zstd_error(zresult) + ) if out_buffer.pos: ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) @@ -2408,48 +2508,47 @@ def decompress_content_dict_chain(self, frames): if not isinstance(frames, list): - raise TypeError('argument must be a list') + raise TypeError("argument must be a list") if not frames: - raise ValueError('empty input chain') + raise ValueError("empty input chain") # First chunk should not be using a dictionary. We handle it specially. chunk = frames[0] if not isinstance(chunk, bytes_type): - raise ValueError('chunk 0 must be bytes') + raise ValueError("chunk 0 must be bytes") # All chunks should be zstd frames and should have content size set. chunk_buffer = ffi.from_buffer(chunk) - params = ffi.new('ZSTD_frameHeader *') + params = ffi.new("ZSTD_frameHeader *") zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer)) if lib.ZSTD_isError(zresult): - raise ValueError('chunk 0 is not a valid zstd frame') + raise ValueError("chunk 0 is not a valid zstd frame") elif zresult: - raise ValueError('chunk 0 is too small to contain a zstd frame') + raise ValueError("chunk 0 is too small to contain a zstd frame") if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN: - raise ValueError('chunk 0 missing content size in frame') + raise ValueError("chunk 0 missing content size in frame") self._ensure_dctx(load_dict=False) - last_buffer = ffi.new('char[]', params.frameContentSize) - - out_buffer = ffi.new('ZSTD_outBuffer *') + last_buffer = ffi.new("char[]", params.frameContentSize) + + out_buffer = ffi.new("ZSTD_outBuffer *") out_buffer.dst = last_buffer out_buffer.size = len(last_buffer) out_buffer.pos = 0 - in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer = ffi.new("ZSTD_inBuffer *") in_buffer.src = chunk_buffer in_buffer.size = len(chunk_buffer) in_buffer.pos = 0 zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) if lib.ZSTD_isError(zresult): - raise ZstdError('could not decompress chunk 0: %s' % - _zstd_error(zresult)) + raise ZstdError("could not decompress chunk 0: %s" % _zstd_error(zresult)) elif zresult: - raise ZstdError('chunk 0 did not decompress full frame') + raise ZstdError("chunk 0 did not decompress full frame") # Special case of chain length of 1 if len(frames) == 1: @@ -2459,19 +2558,19 @@ while i < len(frames): chunk = frames[i] if not isinstance(chunk, bytes_type): - raise ValueError('chunk %d must be bytes' % i) + raise ValueError("chunk %d must be bytes" % i) chunk_buffer = ffi.from_buffer(chunk) zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer)) if lib.ZSTD_isError(zresult): - raise ValueError('chunk %d is not a valid zstd frame' % i) + raise ValueError("chunk %d is not a valid zstd frame" % i) elif zresult: - raise ValueError('chunk %d is too small to contain a zstd frame' % i) + raise ValueError("chunk %d is too small to contain a zstd frame" % i) if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN: - raise ValueError('chunk %d missing content size in frame' % i) - - dest_buffer = ffi.new('char[]', params.frameContentSize) + raise ValueError("chunk %d missing content size in frame" % i) + + dest_buffer = ffi.new("char[]", params.frameContentSize) out_buffer.dst = dest_buffer out_buffer.size = len(dest_buffer) @@ -2483,10 +2582,11 @@ zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) if lib.ZSTD_isError(zresult): - raise ZstdError('could not decompress chunk %d: %s' % - _zstd_error(zresult)) + raise ZstdError( + "could not decompress chunk %d: %s" % _zstd_error(zresult) + ) elif zresult: - raise ZstdError('chunk %d did not decompress full frame' % i) + raise ZstdError("chunk %d did not decompress full frame" % i) last_buffer = dest_buffer i += 1 @@ -2497,19 +2597,19 @@ lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only) if self._max_window_size: - zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx, - self._max_window_size) + zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx, self._max_window_size) if lib.ZSTD_isError(zresult): - raise ZstdError('unable to set max window size: %s' % - _zstd_error(zresult)) + raise ZstdError( + "unable to set max window size: %s" % _zstd_error(zresult) + ) zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format) if lib.ZSTD_isError(zresult): - raise ZstdError('unable to set decoding format: %s' % - _zstd_error(zresult)) + raise ZstdError("unable to set decoding format: %s" % _zstd_error(zresult)) if self._dict_data and load_dict: zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict) if lib.ZSTD_isError(zresult): - raise ZstdError('unable to reference prepared dictionary: %s' % - _zstd_error(zresult)) + raise ZstdError( + "unable to reference prepared dictionary: %s" % _zstd_error(zresult) + )