Mercurial > hg
annotate contrib/python-zstandard/zstandard/cffi.py @ 44147:5e84a96d865b
python-zstandard: blacken at 80 characters
I made this change upstream and it will make it into the next
release of python-zstandard. I figured I'd send it Mercurial's
way because it will allow us to drop this directory from the black
exclusion list.
# skip-blame blackening
Differential Revision: https://phab.mercurial-scm.org/D7937
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Wed, 22 Jan 2020 22:23:04 -0800 |
parents | de7838053207 |
children | 6000f5b25c9b |
rev | line source |
---|---|
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # Copyright (c) 2016-present, Gregory Szorc |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # All rights reserved. |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # This software may be modified and distributed under the terms |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # of the BSD license. See the LICENSE file for details. |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 """Python interface to the Zstandard (zstd) compression library.""" |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 from __future__ import absolute_import, unicode_literals |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
10 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
11 # This should match what the C extension exports. |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
12 __all__ = [ |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
13 #'BufferSegment', |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
14 #'BufferSegments', |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
15 #'BufferWithSegments', |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
16 #'BufferWithSegmentsCollection', |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
17 "CompressionParameters", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
18 "ZstdCompressionDict", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
19 "ZstdCompressionParameters", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
20 "ZstdCompressor", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
21 "ZstdError", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
22 "ZstdDecompressor", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
23 "FrameParameters", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
24 "estimate_decompression_context_size", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
25 "frame_content_size", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
26 "frame_header_size", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
27 "get_frame_parameters", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
28 "train_dictionary", |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
29 # Constants. |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
30 "FLUSH_BLOCK", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
31 "FLUSH_FRAME", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
32 "COMPRESSOBJ_FLUSH_FINISH", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
33 "COMPRESSOBJ_FLUSH_BLOCK", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
34 "ZSTD_VERSION", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
35 "FRAME_HEADER", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
36 "CONTENTSIZE_UNKNOWN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
37 "CONTENTSIZE_ERROR", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
38 "MAX_COMPRESSION_LEVEL", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
39 "COMPRESSION_RECOMMENDED_INPUT_SIZE", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
40 "COMPRESSION_RECOMMENDED_OUTPUT_SIZE", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
41 "DECOMPRESSION_RECOMMENDED_INPUT_SIZE", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
42 "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
43 "MAGIC_NUMBER", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
44 "BLOCKSIZELOG_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
45 "BLOCKSIZE_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
46 "WINDOWLOG_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
47 "WINDOWLOG_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
48 "CHAINLOG_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
49 "CHAINLOG_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
50 "HASHLOG_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
51 "HASHLOG_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
52 "HASHLOG3_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
53 "MINMATCH_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
54 "MINMATCH_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
55 "SEARCHLOG_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
56 "SEARCHLOG_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
57 "SEARCHLENGTH_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
58 "SEARCHLENGTH_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
59 "TARGETLENGTH_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
60 "TARGETLENGTH_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
61 "LDM_MINMATCH_MIN", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
62 "LDM_MINMATCH_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
63 "LDM_BUCKETSIZELOG_MAX", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
64 "STRATEGY_FAST", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
65 "STRATEGY_DFAST", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
66 "STRATEGY_GREEDY", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
67 "STRATEGY_LAZY", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
68 "STRATEGY_LAZY2", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
69 "STRATEGY_BTLAZY2", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
70 "STRATEGY_BTOPT", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
71 "STRATEGY_BTULTRA", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
72 "STRATEGY_BTULTRA2", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
73 "DICT_TYPE_AUTO", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
74 "DICT_TYPE_RAWCONTENT", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
75 "DICT_TYPE_FULLDICT", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
76 "FORMAT_ZSTD1", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
77 "FORMAT_ZSTD1_MAGICLESS", |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
78 ] |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
79 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
80 import io |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
81 import os |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
82 import sys |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
83 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
84 from _zstd_cffi import ( |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
85 ffi, |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
86 lib, |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
87 ) |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
88 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
89 if sys.version_info[0] == 2: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
90 bytes_type = str |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
91 int_type = long |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
92 else: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
93 bytes_type = bytes |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
94 int_type = int |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
95 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
96 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
97 COMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_CStreamInSize() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
98 COMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_CStreamOutSize() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
99 DECOMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_DStreamInSize() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
100 DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_DStreamOutSize() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
101 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
102 new_nonzero = ffi.new_allocator(should_clear_after_alloc=False) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
103 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
104 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
105 MAX_COMPRESSION_LEVEL = lib.ZSTD_maxCLevel() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
106 MAGIC_NUMBER = lib.ZSTD_MAGICNUMBER |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
107 FRAME_HEADER = b"\x28\xb5\x2f\xfd" |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
108 CONTENTSIZE_UNKNOWN = lib.ZSTD_CONTENTSIZE_UNKNOWN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
109 CONTENTSIZE_ERROR = lib.ZSTD_CONTENTSIZE_ERROR |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
110 ZSTD_VERSION = ( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
111 lib.ZSTD_VERSION_MAJOR, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
112 lib.ZSTD_VERSION_MINOR, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
113 lib.ZSTD_VERSION_RELEASE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
114 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
115 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
116 BLOCKSIZELOG_MAX = lib.ZSTD_BLOCKSIZELOG_MAX |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
117 BLOCKSIZE_MAX = lib.ZSTD_BLOCKSIZE_MAX |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
118 WINDOWLOG_MIN = lib.ZSTD_WINDOWLOG_MIN |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
119 WINDOWLOG_MAX = lib.ZSTD_WINDOWLOG_MAX |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
120 CHAINLOG_MIN = lib.ZSTD_CHAINLOG_MIN |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
121 CHAINLOG_MAX = lib.ZSTD_CHAINLOG_MAX |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
122 HASHLOG_MIN = lib.ZSTD_HASHLOG_MIN |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
123 HASHLOG_MAX = lib.ZSTD_HASHLOG_MAX |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
124 HASHLOG3_MAX = lib.ZSTD_HASHLOG3_MAX |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
125 MINMATCH_MIN = lib.ZSTD_MINMATCH_MIN |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
126 MINMATCH_MAX = lib.ZSTD_MINMATCH_MAX |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
127 SEARCHLOG_MIN = lib.ZSTD_SEARCHLOG_MIN |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
128 SEARCHLOG_MAX = lib.ZSTD_SEARCHLOG_MAX |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
129 SEARCHLENGTH_MIN = lib.ZSTD_MINMATCH_MIN |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
130 SEARCHLENGTH_MAX = lib.ZSTD_MINMATCH_MAX |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
131 TARGETLENGTH_MIN = lib.ZSTD_TARGETLENGTH_MIN |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
132 TARGETLENGTH_MAX = lib.ZSTD_TARGETLENGTH_MAX |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
133 LDM_MINMATCH_MIN = lib.ZSTD_LDM_MINMATCH_MIN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
134 LDM_MINMATCH_MAX = lib.ZSTD_LDM_MINMATCH_MAX |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
135 LDM_BUCKETSIZELOG_MAX = lib.ZSTD_LDM_BUCKETSIZELOG_MAX |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
136 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
137 STRATEGY_FAST = lib.ZSTD_fast |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
138 STRATEGY_DFAST = lib.ZSTD_dfast |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
139 STRATEGY_GREEDY = lib.ZSTD_greedy |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
140 STRATEGY_LAZY = lib.ZSTD_lazy |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
141 STRATEGY_LAZY2 = lib.ZSTD_lazy2 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
142 STRATEGY_BTLAZY2 = lib.ZSTD_btlazy2 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
143 STRATEGY_BTOPT = lib.ZSTD_btopt |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
144 STRATEGY_BTULTRA = lib.ZSTD_btultra |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
145 STRATEGY_BTULTRA2 = lib.ZSTD_btultra2 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
146 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
147 DICT_TYPE_AUTO = lib.ZSTD_dct_auto |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
148 DICT_TYPE_RAWCONTENT = lib.ZSTD_dct_rawContent |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
149 DICT_TYPE_FULLDICT = lib.ZSTD_dct_fullDict |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
150 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
151 FORMAT_ZSTD1 = lib.ZSTD_f_zstd1 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
152 FORMAT_ZSTD1_MAGICLESS = lib.ZSTD_f_zstd1_magicless |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
153 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
154 FLUSH_BLOCK = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
155 FLUSH_FRAME = 1 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
156 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
157 COMPRESSOBJ_FLUSH_FINISH = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
158 COMPRESSOBJ_FLUSH_BLOCK = 1 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
159 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
160 |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
161 def _cpu_count(): |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
162 # os.cpu_count() was introducd in Python 3.4. |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
163 try: |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
164 return os.cpu_count() or 0 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
165 except AttributeError: |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
166 pass |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
167 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
168 # Linux. |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
169 try: |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
170 if sys.version_info[0] == 2: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
171 return os.sysconf(b"SC_NPROCESSORS_ONLN") |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
172 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
173 return os.sysconf("SC_NPROCESSORS_ONLN") |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
174 except (AttributeError, ValueError): |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
175 pass |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
176 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
177 # TODO implement on other platforms. |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
178 return 0 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
179 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
180 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
181 class ZstdError(Exception): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
182 pass |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
183 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
184 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
185 def _zstd_error(zresult): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
186 # Resolves to bytes on Python 2 and 3. We use the string for formatting |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
187 # into error messages, which will be literal unicode. So convert it to |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
188 # unicode. |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
189 return ffi.string(lib.ZSTD_getErrorName(zresult)).decode("utf-8") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
190 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
191 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
192 def _make_cctx_params(params): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
193 res = lib.ZSTD_createCCtxParams() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
194 if res == ffi.NULL: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
195 raise MemoryError() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
196 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
197 res = ffi.gc(res, lib.ZSTD_freeCCtxParams) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
198 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
199 attrs = [ |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
200 (lib.ZSTD_c_format, params.format), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
201 (lib.ZSTD_c_compressionLevel, params.compression_level), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
202 (lib.ZSTD_c_windowLog, params.window_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
203 (lib.ZSTD_c_hashLog, params.hash_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
204 (lib.ZSTD_c_chainLog, params.chain_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
205 (lib.ZSTD_c_searchLog, params.search_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
206 (lib.ZSTD_c_minMatch, params.min_match), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
207 (lib.ZSTD_c_targetLength, params.target_length), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
208 (lib.ZSTD_c_strategy, params.compression_strategy), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
209 (lib.ZSTD_c_contentSizeFlag, params.write_content_size), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
210 (lib.ZSTD_c_checksumFlag, params.write_checksum), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
211 (lib.ZSTD_c_dictIDFlag, params.write_dict_id), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
212 (lib.ZSTD_c_nbWorkers, params.threads), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
213 (lib.ZSTD_c_jobSize, params.job_size), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
214 (lib.ZSTD_c_overlapLog, params.overlap_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
215 (lib.ZSTD_c_forceMaxWindow, params.force_max_window), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
216 (lib.ZSTD_c_enableLongDistanceMatching, params.enable_ldm), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
217 (lib.ZSTD_c_ldmHashLog, params.ldm_hash_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
218 (lib.ZSTD_c_ldmMinMatch, params.ldm_min_match), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
219 (lib.ZSTD_c_ldmBucketSizeLog, params.ldm_bucket_size_log), |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
220 (lib.ZSTD_c_ldmHashRateLog, params.ldm_hash_rate_log), |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
221 ] |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
222 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
223 for param, value in attrs: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
224 _set_compression_parameter(res, param, value) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
225 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
226 return res |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
227 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
228 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
229 class ZstdCompressionParameters(object): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
230 @staticmethod |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
231 def from_level(level, source_size=0, dict_size=0, **kwargs): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
232 params = lib.ZSTD_getCParams(level, source_size, dict_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
233 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
234 args = { |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
235 "window_log": "windowLog", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
236 "chain_log": "chainLog", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
237 "hash_log": "hashLog", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
238 "search_log": "searchLog", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
239 "min_match": "minMatch", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
240 "target_length": "targetLength", |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
241 "compression_strategy": "strategy", |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
242 } |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
243 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
244 for arg, attr in args.items(): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
245 if arg not in kwargs: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
246 kwargs[arg] = getattr(params, attr) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
247 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
248 return ZstdCompressionParameters(**kwargs) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
249 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
250 def __init__( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
251 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
252 format=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
253 compression_level=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
254 window_log=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
255 hash_log=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
256 chain_log=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
257 search_log=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
258 min_match=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
259 target_length=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
260 strategy=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
261 compression_strategy=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
262 write_content_size=1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
263 write_checksum=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
264 write_dict_id=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
265 job_size=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
266 overlap_log=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
267 overlap_size_log=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
268 force_max_window=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
269 enable_ldm=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
270 ldm_hash_log=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
271 ldm_min_match=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
272 ldm_bucket_size_log=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
273 ldm_hash_rate_log=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
274 ldm_hash_every_log=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
275 threads=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
276 ): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
277 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
278 params = lib.ZSTD_createCCtxParams() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
279 if params == ffi.NULL: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
280 raise MemoryError() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
281 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
282 params = ffi.gc(params, lib.ZSTD_freeCCtxParams) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
283 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
284 self._params = params |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
285 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
286 if threads < 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
287 threads = _cpu_count() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
288 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
289 # We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
290 # because setting ZSTD_c_nbWorkers resets the other parameters. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
291 _set_compression_parameter(params, lib.ZSTD_c_nbWorkers, threads) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
292 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
293 _set_compression_parameter(params, lib.ZSTD_c_format, format) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
294 _set_compression_parameter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
295 params, lib.ZSTD_c_compressionLevel, compression_level |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
296 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
297 _set_compression_parameter(params, lib.ZSTD_c_windowLog, window_log) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
298 _set_compression_parameter(params, lib.ZSTD_c_hashLog, hash_log) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
299 _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
300 _set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
301 _set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match) |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
302 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
303 params, lib.ZSTD_c_targetLength, target_length |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
304 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
305 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
306 if strategy != -1 and compression_strategy != -1: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
307 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
308 "cannot specify both compression_strategy and strategy" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
309 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
310 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
311 if compression_strategy != -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
312 strategy = compression_strategy |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
313 elif strategy == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
314 strategy = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
315 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
316 _set_compression_parameter(params, lib.ZSTD_c_strategy, strategy) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
317 _set_compression_parameter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
318 params, lib.ZSTD_c_contentSizeFlag, write_content_size |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
319 ) |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
320 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
321 params, lib.ZSTD_c_checksumFlag, write_checksum |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
322 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
323 _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
324 _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
325 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
326 if overlap_log != -1 and overlap_size_log != -1: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
327 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
328 "cannot specify both overlap_log and overlap_size_log" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
329 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
330 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
331 if overlap_size_log != -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
332 overlap_log = overlap_size_log |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
333 elif overlap_log == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
334 overlap_log = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
335 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
336 _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log) |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
337 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
338 params, lib.ZSTD_c_forceMaxWindow, force_max_window |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
339 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
340 _set_compression_parameter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
341 params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
342 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
343 _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log) |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
344 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
345 params, lib.ZSTD_c_ldmMinMatch, ldm_min_match |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
346 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
347 _set_compression_parameter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
348 params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
349 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
350 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
351 if ldm_hash_rate_log != -1 and ldm_hash_every_log != -1: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
352 raise ValueError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
353 "cannot specify both ldm_hash_rate_log and ldm_hash_every_log" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
354 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
355 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
356 if ldm_hash_every_log != -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
357 ldm_hash_rate_log = ldm_hash_every_log |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
358 elif ldm_hash_rate_log == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
359 ldm_hash_rate_log = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
360 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
361 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
362 params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
363 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
364 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
365 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
366 def format(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
367 return _get_compression_parameter(self._params, lib.ZSTD_c_format) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
368 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
369 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
370 def compression_level(self): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
371 return _get_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
372 self._params, lib.ZSTD_c_compressionLevel |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
373 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
374 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
375 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
376 def window_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
377 return _get_compression_parameter(self._params, lib.ZSTD_c_windowLog) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
378 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
379 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
380 def hash_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
381 return _get_compression_parameter(self._params, lib.ZSTD_c_hashLog) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
382 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
383 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
384 def chain_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
385 return _get_compression_parameter(self._params, lib.ZSTD_c_chainLog) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
386 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
387 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
388 def search_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
389 return _get_compression_parameter(self._params, lib.ZSTD_c_searchLog) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
390 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
391 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
392 def min_match(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
393 return _get_compression_parameter(self._params, lib.ZSTD_c_minMatch) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
394 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
395 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
396 def target_length(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
397 return _get_compression_parameter(self._params, lib.ZSTD_c_targetLength) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
398 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
399 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
400 def compression_strategy(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
401 return _get_compression_parameter(self._params, lib.ZSTD_c_strategy) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
402 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
403 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
404 def write_content_size(self): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
405 return _get_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
406 self._params, lib.ZSTD_c_contentSizeFlag |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
407 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
408 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
409 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
410 def write_checksum(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
411 return _get_compression_parameter(self._params, lib.ZSTD_c_checksumFlag) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
412 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
413 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
414 def write_dict_id(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
415 return _get_compression_parameter(self._params, lib.ZSTD_c_dictIDFlag) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
416 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
417 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
418 def job_size(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
419 return _get_compression_parameter(self._params, lib.ZSTD_c_jobSize) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
420 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
421 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
422 def overlap_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
423 return _get_compression_parameter(self._params, lib.ZSTD_c_overlapLog) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
424 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
425 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
426 def overlap_size_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
427 return self.overlap_log |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
428 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
429 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
430 def force_max_window(self): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
431 return _get_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
432 self._params, lib.ZSTD_c_forceMaxWindow |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
433 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
434 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
435 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
436 def enable_ldm(self): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
437 return _get_compression_parameter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
438 self._params, lib.ZSTD_c_enableLongDistanceMatching |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
439 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
440 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
441 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
442 def ldm_hash_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
443 return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashLog) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
444 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
445 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
446 def ldm_min_match(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
447 return _get_compression_parameter(self._params, lib.ZSTD_c_ldmMinMatch) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
448 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
449 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
450 def ldm_bucket_size_log(self): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
451 return _get_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
452 self._params, lib.ZSTD_c_ldmBucketSizeLog |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
453 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
454 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
455 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
456 def ldm_hash_rate_log(self): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
457 return _get_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
458 self._params, lib.ZSTD_c_ldmHashRateLog |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
459 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
460 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
461 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
462 def ldm_hash_every_log(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
463 return self.ldm_hash_rate_log |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
464 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
465 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
466 def threads(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
467 return _get_compression_parameter(self._params, lib.ZSTD_c_nbWorkers) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
468 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
469 def estimated_compression_context_size(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
470 return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self._params) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
471 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
472 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
473 CompressionParameters = ZstdCompressionParameters |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
474 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
475 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
476 def estimate_decompression_context_size(): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
477 return lib.ZSTD_estimateDCtxSize() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
478 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
479 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
480 def _set_compression_parameter(params, param, value): |
42937
69de49c4e39c
zstandard: vendor python-zstandard 0.12
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42070
diff
changeset
|
481 zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
482 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
483 raise ZstdError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
484 "unable to set compression context parameter: %s" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
485 % _zstd_error(zresult) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
486 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
487 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
488 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
489 def _get_compression_parameter(params, param): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
490 result = ffi.new("int *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
491 |
42937
69de49c4e39c
zstandard: vendor python-zstandard 0.12
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42070
diff
changeset
|
492 zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
493 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
494 raise ZstdError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
495 "unable to get compression context parameter: %s" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
496 % _zstd_error(zresult) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
497 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
498 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
499 return result[0] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
500 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
501 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
502 class ZstdCompressionWriter(object): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
503 def __init__( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
504 self, compressor, writer, source_size, write_size, write_return_read |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
505 ): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
506 self._compressor = compressor |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
507 self._writer = writer |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
508 self._write_size = write_size |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
509 self._write_return_read = bool(write_return_read) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
510 self._entered = False |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
511 self._closed = False |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
512 self._bytes_compressed = 0 |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
513 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
514 self._dst_buffer = ffi.new("char[]", write_size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
515 self._out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
516 self._out_buffer.dst = self._dst_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
517 self._out_buffer.size = len(self._dst_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
518 self._out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
519 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
520 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, source_size) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
521 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
522 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
523 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
524 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
525 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
526 def __enter__(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
527 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
528 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
529 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
530 if self._entered: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
531 raise ZstdError("cannot __enter__ multiple times") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
532 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
533 self._entered = True |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
534 return self |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
535 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
536 def __exit__(self, exc_type, exc_value, exc_tb): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
537 self._entered = False |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
538 |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
539 if not exc_type and not exc_value and not exc_tb: |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
540 self.close() |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
541 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
542 self._compressor = None |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
543 |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
544 return False |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
545 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
546 def memory_size(self): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
547 return lib.ZSTD_sizeof_CCtx(self._compressor._cctx) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
548 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
549 def fileno(self): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
550 f = getattr(self._writer, "fileno", None) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
551 if f: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
552 return f() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
553 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
554 raise OSError("fileno not available on underlying writer") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
555 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
556 def close(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
557 if self._closed: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
558 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
559 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
560 try: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
561 self.flush(FLUSH_FRAME) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
562 finally: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
563 self._closed = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
564 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
565 # Call close() on underlying stream as well. |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
566 f = getattr(self._writer, "close", None) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
567 if f: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
568 f() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
569 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
570 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
571 def closed(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
572 return self._closed |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
573 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
574 def isatty(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
575 return False |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
576 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
577 def readable(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
578 return False |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
579 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
580 def readline(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
581 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
582 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
583 def readlines(self, hint=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
584 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
585 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
586 def seek(self, offset, whence=None): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
587 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
588 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
589 def seekable(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
590 return False |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
591 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
592 def truncate(self, size=None): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
593 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
594 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
595 def writable(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
596 return True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
597 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
598 def writelines(self, lines): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
599 raise NotImplementedError("writelines() is not yet implemented") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
600 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
601 def read(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
602 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
603 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
604 def readall(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
605 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
606 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
607 def readinto(self, b): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
608 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
609 |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
610 def write(self, data): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
611 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
612 raise ValueError("stream is closed") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
613 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
614 total_write = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
615 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
616 data_buffer = ffi.from_buffer(data) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
617 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
618 in_buffer = ffi.new("ZSTD_inBuffer *") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
619 in_buffer.src = data_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
620 in_buffer.size = len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
621 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
622 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
623 out_buffer = self._out_buffer |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
624 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
625 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
626 while in_buffer.pos < in_buffer.size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
627 zresult = lib.ZSTD_compressStream2( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
628 self._compressor._cctx, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
629 out_buffer, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
630 in_buffer, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
631 lib.ZSTD_e_continue, |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
632 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
633 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
634 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
635 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
636 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
637 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
638 if out_buffer.pos: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
639 self._writer.write( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
640 ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
641 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
642 total_write += out_buffer.pos |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
643 self._bytes_compressed += out_buffer.pos |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
644 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
645 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
646 if self._write_return_read: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
647 return in_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
648 else: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
649 return total_write |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
650 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
651 def flush(self, flush_mode=FLUSH_BLOCK): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
652 if flush_mode == FLUSH_BLOCK: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
653 flush = lib.ZSTD_e_flush |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
654 elif flush_mode == FLUSH_FRAME: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
655 flush = lib.ZSTD_e_end |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
656 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
657 raise ValueError("unknown flush_mode: %r" % flush_mode) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
658 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
659 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
660 raise ValueError("stream is closed") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
661 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
662 total_write = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
663 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
664 out_buffer = self._out_buffer |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
665 out_buffer.pos = 0 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
666 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
667 in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
668 in_buffer.src = ffi.NULL |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
669 in_buffer.size = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
670 in_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
671 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
672 while True: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
673 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
674 self._compressor._cctx, out_buffer, in_buffer, flush |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
675 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
676 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
677 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
678 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
679 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
680 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
681 if out_buffer.pos: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
682 self._writer.write( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
683 ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
684 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
685 total_write += out_buffer.pos |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
686 self._bytes_compressed += out_buffer.pos |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
687 out_buffer.pos = 0 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
688 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
689 if not zresult: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
690 break |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
691 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
692 return total_write |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
693 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
694 def tell(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
695 return self._bytes_compressed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
696 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
697 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
698 class ZstdCompressionObj(object): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
699 def compress(self, data): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
700 if self._finished: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
701 raise ZstdError("cannot call compress() after compressor finished") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
702 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
703 data_buffer = ffi.from_buffer(data) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
704 source = ffi.new("ZSTD_inBuffer *") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
705 source.src = data_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
706 source.size = len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
707 source.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
708 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
709 chunks = [] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
710 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
711 while source.pos < len(data): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
712 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
713 self._compressor._cctx, self._out, source, lib.ZSTD_e_continue |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
714 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
715 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
716 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
717 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
718 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
719 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
720 if self._out.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
721 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:]) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
722 self._out.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
723 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
724 return b"".join(chunks) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
725 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
726 def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
727 if flush_mode not in ( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
728 COMPRESSOBJ_FLUSH_FINISH, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
729 COMPRESSOBJ_FLUSH_BLOCK, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
730 ): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
731 raise ValueError("flush mode not recognized") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
732 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
733 if self._finished: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
734 raise ZstdError("compressor object already finished") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
735 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
736 if flush_mode == COMPRESSOBJ_FLUSH_BLOCK: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
737 z_flush_mode = lib.ZSTD_e_flush |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
738 elif flush_mode == COMPRESSOBJ_FLUSH_FINISH: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
739 z_flush_mode = lib.ZSTD_e_end |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
740 self._finished = True |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
741 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
742 raise ZstdError("unhandled flush mode") |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
743 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
744 assert self._out.pos == 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
745 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
746 in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
747 in_buffer.src = ffi.NULL |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
748 in_buffer.size = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
749 in_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
750 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
751 chunks = [] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
752 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
753 while True: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
754 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
755 self._compressor._cctx, self._out, in_buffer, z_flush_mode |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
756 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
757 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
758 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
759 "error ending compression stream: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
760 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
761 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
762 if self._out.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
763 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:]) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
764 self._out.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
765 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
766 if not zresult: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
767 break |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
768 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
769 return b"".join(chunks) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
770 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
771 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
772 class ZstdCompressionChunker(object): |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
773 def __init__(self, compressor, chunk_size): |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
774 self._compressor = compressor |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
775 self._out = ffi.new("ZSTD_outBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
776 self._dst_buffer = ffi.new("char[]", chunk_size) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
777 self._out.dst = self._dst_buffer |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
778 self._out.size = chunk_size |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
779 self._out.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
780 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
781 self._in = ffi.new("ZSTD_inBuffer *") |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
782 self._in.src = ffi.NULL |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
783 self._in.size = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
784 self._in.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
785 self._finished = False |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
786 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
787 def compress(self, data): |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
788 if self._finished: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
789 raise ZstdError("cannot call compress() after compression finished") |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
790 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
791 if self._in.src != ffi.NULL: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
792 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
793 "cannot perform operation before consuming output " |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
794 "from previous operation" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
795 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
796 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
797 data_buffer = ffi.from_buffer(data) |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
798 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
799 if not len(data_buffer): |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
800 return |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
801 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
802 self._in.src = data_buffer |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
803 self._in.size = len(data_buffer) |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
804 self._in.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
805 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
806 while self._in.pos < self._in.size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
807 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
808 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_continue |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
809 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
810 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
811 if self._in.pos == self._in.size: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
812 self._in.src = ffi.NULL |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
813 self._in.size = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
814 self._in.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
815 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
816 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
817 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
818 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
819 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
820 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
821 if self._out.pos == self._out.size: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
822 yield ffi.buffer(self._out.dst, self._out.pos)[:] |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
823 self._out.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
824 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
825 def flush(self): |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
826 if self._finished: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
827 raise ZstdError("cannot call flush() after compression finished") |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
828 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
829 if self._in.src != ffi.NULL: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
830 raise ZstdError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
831 "cannot call flush() before consuming output from " |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
832 "previous operation" |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
833 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
834 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
835 while True: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
836 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
837 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_flush |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
838 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
839 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
840 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
841 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
842 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
843 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
844 if self._out.pos: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
845 yield ffi.buffer(self._out.dst, self._out.pos)[:] |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
846 self._out.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
847 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
848 if not zresult: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
849 return |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
850 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
851 def finish(self): |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
852 if self._finished: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
853 raise ZstdError("cannot call finish() after compression finished") |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
854 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
855 if self._in.src != ffi.NULL: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
856 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
857 "cannot call finish() before consuming output from " |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
858 "previous operation" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
859 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
860 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
861 while True: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
862 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
863 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
864 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
865 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
866 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
867 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
868 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
869 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
870 if self._out.pos: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
871 yield ffi.buffer(self._out.dst, self._out.pos)[:] |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
872 self._out.pos = 0 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
873 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
874 if not zresult: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
875 self._finished = True |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
876 return |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
877 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
878 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
879 class ZstdCompressionReader(object): |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
880 def __init__(self, compressor, source, read_size): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
881 self._compressor = compressor |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
882 self._source = source |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
883 self._read_size = read_size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
884 self._entered = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
885 self._closed = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
886 self._bytes_compressed = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
887 self._finished_input = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
888 self._finished_output = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
889 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
890 self._in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
891 # Holds a ref so backing bytes in self._in_buffer stay alive. |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
892 self._source_buffer = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
893 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
894 def __enter__(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
895 if self._entered: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
896 raise ValueError("cannot __enter__ multiple times") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
897 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
898 self._entered = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
899 return self |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
900 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
901 def __exit__(self, exc_type, exc_value, exc_tb): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
902 self._entered = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
903 self._closed = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
904 self._source = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
905 self._compressor = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
906 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
907 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
908 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
909 def readable(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
910 return True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
911 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
912 def writable(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
913 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
914 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
915 def seekable(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
916 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
917 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
918 def readline(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
919 raise io.UnsupportedOperation() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
920 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
921 def readlines(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
922 raise io.UnsupportedOperation() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
923 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
924 def write(self, data): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
925 raise OSError("stream is not writable") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
926 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
927 def writelines(self, ignored): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
928 raise OSError("stream is not writable") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
929 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
930 def isatty(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
931 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
932 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
933 def flush(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
934 return None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
935 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
936 def close(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
937 self._closed = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
938 return None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
939 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
940 @property |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
941 def closed(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
942 return self._closed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
943 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
944 def tell(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
945 return self._bytes_compressed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
946 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
947 def readall(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
948 chunks = [] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
949 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
950 while True: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
951 chunk = self.read(1048576) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
952 if not chunk: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
953 break |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
954 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
955 chunks.append(chunk) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
956 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
957 return b"".join(chunks) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
958 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
959 def __iter__(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
960 raise io.UnsupportedOperation() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
961 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
962 def __next__(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
963 raise io.UnsupportedOperation() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
964 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
965 next = __next__ |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
966 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
967 def _read_input(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
968 if self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
969 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
970 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
971 if hasattr(self._source, "read"): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
972 data = self._source.read(self._read_size) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
973 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
974 if not data: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
975 self._finished_input = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
976 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
977 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
978 self._source_buffer = ffi.from_buffer(data) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
979 self._in_buffer.src = self._source_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
980 self._in_buffer.size = len(self._source_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
981 self._in_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
982 else: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
983 self._source_buffer = ffi.from_buffer(self._source) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
984 self._in_buffer.src = self._source_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
985 self._in_buffer.size = len(self._source_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
986 self._in_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
987 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
988 def _compress_into_buffer(self, out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
989 if self._in_buffer.pos >= self._in_buffer.size: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
990 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
991 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
992 old_pos = out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
993 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
994 zresult = lib.ZSTD_compressStream2( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
995 self._compressor._cctx, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
996 out_buffer, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
997 self._in_buffer, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
998 lib.ZSTD_e_continue, |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
999 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1000 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1001 self._bytes_compressed += out_buffer.pos - old_pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1002 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1003 if self._in_buffer.pos == self._in_buffer.size: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1004 self._in_buffer.src = ffi.NULL |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1005 self._in_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1006 self._in_buffer.size = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1007 self._source_buffer = None |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1008 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1009 if not hasattr(self._source, "read"): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1010 self._finished_input = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1011 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1012 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1013 raise ZstdError("zstd compress error: %s", _zstd_error(zresult)) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1014 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1015 return out_buffer.pos and out_buffer.pos == out_buffer.size |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1016 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1017 def read(self, size=-1): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1018 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1019 raise ValueError("stream is closed") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1020 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1021 if size < -1: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1022 raise ValueError("cannot read negative amounts less than -1") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1023 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1024 if size == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1025 return self.readall() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1026 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1027 if self._finished_output or size == 0: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1028 return b"" |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1029 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1030 # Need a dedicated ref to dest buffer otherwise it gets collected. |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1031 dst_buffer = ffi.new("char[]", size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1032 out_buffer = ffi.new("ZSTD_outBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1033 out_buffer.dst = dst_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1034 out_buffer.size = size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1035 out_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1036 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1037 if self._compress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1038 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1039 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1040 while not self._finished_input: |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1041 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1042 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1043 if self._compress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1044 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1045 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1046 # EOF |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1047 old_pos = out_buffer.pos |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1048 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1049 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1050 self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1051 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1052 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1053 self._bytes_compressed += out_buffer.pos - old_pos |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1054 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1055 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1056 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1057 "error ending compression stream: %s", _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1058 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1059 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1060 if zresult == 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1061 self._finished_output = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1062 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1063 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1064 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1065 def read1(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1066 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1067 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1068 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1069 if size < -1: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1070 raise ValueError("cannot read negative amounts less than -1") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1071 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1072 if self._finished_output or size == 0: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1073 return b"" |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1074 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1075 # -1 returns arbitrary number of bytes. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1076 if size == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1077 size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1078 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1079 dst_buffer = ffi.new("char[]", size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1080 out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1081 out_buffer.dst = dst_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1082 out_buffer.size = size |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1083 out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1084 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1085 # read1() dictates that we can perform at most 1 call to the |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1086 # underlying stream to get input. However, we can't satisfy this |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1087 # restriction with compression because not all input generates output. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1088 # It is possible to perform a block flush in order to ensure output. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1089 # But this may not be desirable behavior. So we allow multiple read() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1090 # to the underlying stream. But unlike read(), we stop once we have |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1091 # any output. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1092 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1093 self._compress_into_buffer(out_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1094 if out_buffer.pos: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1095 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1096 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1097 while not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1098 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1099 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1100 # If we've filled the output buffer, return immediately. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1101 if self._compress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1102 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1103 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1104 # If we've populated the output buffer and we're not at EOF, |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1105 # also return, as we've satisfied the read1() limits. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1106 if out_buffer.pos and not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1107 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1108 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1109 # Else if we're at EOS and we have room left in the buffer, |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1110 # fall through to below and try to add more data to the output. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1111 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1112 # EOF. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1113 old_pos = out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1114 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1115 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1116 self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1117 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1118 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1119 self._bytes_compressed += out_buffer.pos - old_pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1120 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1121 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1122 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1123 "error ending compression stream: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1124 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1125 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1126 if zresult == 0: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1127 self._finished_output = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1128 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1129 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1130 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1131 def readinto(self, b): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1132 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1133 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1134 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1135 if self._finished_output: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1136 return 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1137 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1138 # TODO use writable=True once we require CFFI >= 1.12. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1139 dest_buffer = ffi.from_buffer(b) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1140 ffi.memmove(b, b"", 0) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1141 out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1142 out_buffer.dst = dest_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1143 out_buffer.size = len(dest_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1144 out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1145 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1146 if self._compress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1147 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1148 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1149 while not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1150 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1151 if self._compress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1152 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1153 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1154 # EOF. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1155 old_pos = out_buffer.pos |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1156 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1157 self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1158 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1159 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1160 self._bytes_compressed += out_buffer.pos - old_pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1161 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1162 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1163 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1164 "error ending compression stream: %s", _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1165 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1166 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1167 if zresult == 0: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1168 self._finished_output = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1169 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1170 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1171 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1172 def readinto1(self, b): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1173 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1174 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1175 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1176 if self._finished_output: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1177 return 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1178 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1179 # TODO use writable=True once we require CFFI >= 1.12. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1180 dest_buffer = ffi.from_buffer(b) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1181 ffi.memmove(b, b"", 0) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1182 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1183 out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1184 out_buffer.dst = dest_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1185 out_buffer.size = len(dest_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1186 out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1187 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1188 self._compress_into_buffer(out_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1189 if out_buffer.pos: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1190 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1191 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1192 while not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1193 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1194 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1195 if self._compress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1196 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1197 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1198 if out_buffer.pos and not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1199 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1200 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1201 # EOF. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1202 old_pos = out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1203 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1204 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1205 self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1206 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1207 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1208 self._bytes_compressed += out_buffer.pos - old_pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1209 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1210 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1211 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1212 "error ending compression stream: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1213 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1214 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1215 if zresult == 0: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1216 self._finished_output = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1217 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1218 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1219 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1220 |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1221 class ZstdCompressor(object): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1222 def __init__( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1223 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1224 level=3, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1225 dict_data=None, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1226 compression_params=None, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1227 write_checksum=None, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1228 write_content_size=None, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1229 write_dict_id=None, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1230 threads=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1231 ): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1232 if level > lib.ZSTD_maxCLevel(): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1233 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1234 "level must be less than %d" % lib.ZSTD_maxCLevel() |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1235 ) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1236 |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1237 if threads < 0: |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1238 threads = _cpu_count() |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1239 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1240 if compression_params and write_checksum is not None: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1241 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1242 "cannot define compression_params and " "write_checksum" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1243 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1244 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1245 if compression_params and write_content_size is not None: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1246 raise ValueError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1247 "cannot define compression_params and " "write_content_size" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1248 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1249 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1250 if compression_params and write_dict_id is not None: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1251 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1252 "cannot define compression_params and " "write_dict_id" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1253 ) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1254 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1255 if compression_params and threads: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1256 raise ValueError("cannot define compression_params and threads") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1257 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1258 if compression_params: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1259 self._params = _make_cctx_params(compression_params) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1260 else: |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1261 if write_dict_id is None: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1262 write_dict_id = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1263 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1264 params = lib.ZSTD_createCCtxParams() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1265 if params == ffi.NULL: |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1266 raise MemoryError() |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1267 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1268 self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1269 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1270 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1271 self._params, lib.ZSTD_c_compressionLevel, level |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1272 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1273 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1274 _set_compression_parameter( |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1275 self._params, |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1276 lib.ZSTD_c_contentSizeFlag, |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1277 write_content_size if write_content_size is not None else 1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1278 ) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1279 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1280 _set_compression_parameter( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1281 self._params, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1282 lib.ZSTD_c_checksumFlag, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1283 1 if write_checksum else 0, |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1284 ) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1285 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1286 _set_compression_parameter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1287 self._params, lib.ZSTD_c_dictIDFlag, 1 if write_dict_id else 0 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1288 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1289 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1290 if threads: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1291 _set_compression_parameter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1292 self._params, lib.ZSTD_c_nbWorkers, threads |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1293 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1294 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1295 cctx = lib.ZSTD_createCCtx() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1296 if cctx == ffi.NULL: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1297 raise MemoryError() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1298 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1299 self._cctx = cctx |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1300 self._dict_data = dict_data |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1301 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1302 # We defer setting up garbage collection until after calling |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1303 # _setup_cctx() to ensure the memory size estimate is more accurate. |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1304 try: |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1305 self._setup_cctx() |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1306 finally: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1307 self._cctx = ffi.gc( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1308 cctx, lib.ZSTD_freeCCtx, size=lib.ZSTD_sizeof_CCtx(cctx) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1309 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1310 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1311 def _setup_cctx(self): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1312 zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1313 self._cctx, self._params |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1314 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1315 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1316 raise ZstdError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1317 "could not set compression parameters: %s" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1318 % _zstd_error(zresult) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1319 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1320 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1321 dict_data = self._dict_data |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1322 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1323 if dict_data: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1324 if dict_data._cdict: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1325 zresult = lib.ZSTD_CCtx_refCDict(self._cctx, dict_data._cdict) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1326 else: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1327 zresult = lib.ZSTD_CCtx_loadDictionary_advanced( |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1328 self._cctx, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1329 dict_data.as_bytes(), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1330 len(dict_data), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1331 lib.ZSTD_dlm_byRef, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1332 dict_data._dict_type, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1333 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1334 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1335 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1336 raise ZstdError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1337 "could not load compression dictionary: %s" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1338 % _zstd_error(zresult) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1339 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1340 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1341 def memory_size(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1342 return lib.ZSTD_sizeof_CCtx(self._cctx) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1343 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1344 def compress(self, data): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1345 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1346 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1347 data_buffer = ffi.from_buffer(data) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1348 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1349 dest_size = lib.ZSTD_compressBound(len(data_buffer)) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1350 out = new_nonzero("char[]", dest_size) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1351 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1352 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer)) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1353 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1354 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1355 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1356 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1357 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1358 out_buffer = ffi.new("ZSTD_outBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1359 in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1360 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1361 out_buffer.dst = out |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1362 out_buffer.size = dest_size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1363 out_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1364 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1365 in_buffer.src = data_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1366 in_buffer.size = len(data_buffer) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1367 in_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1368 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1369 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1370 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1371 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1372 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1373 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1374 raise ZstdError("cannot compress: %s" % _zstd_error(zresult)) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1375 elif zresult: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1376 raise ZstdError("unexpected partial frame flush") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1377 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1378 return ffi.buffer(out, out_buffer.pos)[:] |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1379 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1380 def compressobj(self, size=-1): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1381 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1382 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1383 if size < 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1384 size = lib.ZSTD_CONTENTSIZE_UNKNOWN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1385 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1386 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1387 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1388 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1389 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1390 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1391 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1392 cobj = ZstdCompressionObj() |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1393 cobj._out = ffi.new("ZSTD_outBuffer *") |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1394 cobj._dst_buffer = ffi.new( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1395 "char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1396 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1397 cobj._out.dst = cobj._dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1398 cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1399 cobj._out.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1400 cobj._compressor = self |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1401 cobj._finished = False |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1402 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1403 return cobj |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1404 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1405 def chunker(self, size=-1, chunk_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1406 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1407 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1408 if size < 0: |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1409 size = lib.ZSTD_CONTENTSIZE_UNKNOWN |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1410 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1411 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1412 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1413 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1414 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1415 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1416 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1417 return ZstdCompressionChunker(self, chunk_size=chunk_size) |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1418 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1419 def copy_stream( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1420 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1421 ifh, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1422 ofh, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1423 size=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1424 read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1425 write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1426 ): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1427 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1428 if not hasattr(ifh, "read"): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1429 raise ValueError("first argument must have a read() method") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1430 if not hasattr(ofh, "write"): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1431 raise ValueError("second argument must have a write() method") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1432 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1433 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1434 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1435 if size < 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1436 size = lib.ZSTD_CONTENTSIZE_UNKNOWN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1437 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1438 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1439 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1440 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1441 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1442 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1443 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1444 in_buffer = ffi.new("ZSTD_inBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1445 out_buffer = ffi.new("ZSTD_outBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1446 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1447 dst_buffer = ffi.new("char[]", write_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1448 out_buffer.dst = dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1449 out_buffer.size = write_size |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1450 out_buffer.pos = 0 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1451 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1452 total_read, total_write = 0, 0 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1453 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1454 while True: |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1455 data = ifh.read(read_size) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1456 if not data: |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1457 break |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1458 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1459 data_buffer = ffi.from_buffer(data) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1460 total_read += len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1461 in_buffer.src = data_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1462 in_buffer.size = len(data_buffer) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1463 in_buffer.pos = 0 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1464 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1465 while in_buffer.pos < in_buffer.size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1466 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1467 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1468 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1469 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1470 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1471 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1472 ) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1473 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1474 if out_buffer.pos: |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1475 ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1476 total_write += out_buffer.pos |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1477 out_buffer.pos = 0 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1478 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1479 # We've finished reading. Flush the compressor. |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1480 while True: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1481 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1482 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1483 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1484 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1485 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1486 "error ending compression stream: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1487 ) |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1488 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1489 if out_buffer.pos: |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1490 ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1491 total_write += out_buffer.pos |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1492 out_buffer.pos = 0 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1493 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1494 if zresult == 0: |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1495 break |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1496 |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1497 return total_read, total_write |
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1498 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1499 def stream_reader( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1500 self, source, size=-1, read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1501 ): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1502 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1503 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1504 try: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1505 size = len(source) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1506 except Exception: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1507 pass |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1508 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1509 if size < 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1510 size = lib.ZSTD_CONTENTSIZE_UNKNOWN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1511 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1512 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1513 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1514 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1515 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1516 ) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1517 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1518 return ZstdCompressionReader(self, source, read_size) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1519 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1520 def stream_writer( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1521 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1522 writer, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1523 size=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1524 write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1525 write_return_read=False, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1526 ): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1527 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1528 if not hasattr(writer, "write"): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1529 raise ValueError("must pass an object with a write() method") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1530 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1531 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1532 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1533 if size < 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1534 size = lib.ZSTD_CONTENTSIZE_UNKNOWN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1535 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1536 return ZstdCompressionWriter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1537 self, writer, size, write_size, write_return_read |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1538 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1539 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1540 write_to = stream_writer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1541 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1542 def read_to_iter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1543 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1544 reader, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1545 size=-1, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1546 read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1547 write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1548 ): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1549 if hasattr(reader, "read"): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1550 have_read = True |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1551 elif hasattr(reader, "__getitem__"): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1552 have_read = False |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1553 buffer_offset = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1554 size = len(reader) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1555 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1556 raise ValueError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1557 "must pass an object with a read() method or " |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1558 "conforms to buffer protocol" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1559 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1560 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1561 lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1562 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1563 if size < 0: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1564 size = lib.ZSTD_CONTENTSIZE_UNKNOWN |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1565 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1566 zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1567 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1568 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1569 "error setting source size: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1570 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1571 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1572 in_buffer = ffi.new("ZSTD_inBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1573 out_buffer = ffi.new("ZSTD_outBuffer *") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1574 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1575 in_buffer.src = ffi.NULL |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1576 in_buffer.size = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1577 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1578 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1579 dst_buffer = ffi.new("char[]", write_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1580 out_buffer.dst = dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1581 out_buffer.size = write_size |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1582 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1583 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1584 while True: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1585 # We should never have output data sitting around after a previous |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1586 # iteration. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1587 assert out_buffer.pos == 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1588 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1589 # Collect input data. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1590 if have_read: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1591 read_result = reader.read(read_size) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1592 else: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1593 remaining = len(reader) - buffer_offset |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1594 slice_size = min(remaining, read_size) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1595 read_result = reader[buffer_offset : buffer_offset + slice_size] |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1596 buffer_offset += slice_size |
30435
b86a448a2965
zstd: vendor python-zstandard 0.5.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1597 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1598 # No new input data. Break out of the read loop. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1599 if not read_result: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1600 break |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1601 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1602 # Feed all read data into the compressor and emit output until |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1603 # exhausted. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1604 read_buffer = ffi.from_buffer(read_result) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1605 in_buffer.src = read_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1606 in_buffer.size = len(read_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1607 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1608 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1609 while in_buffer.pos < in_buffer.size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1610 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1611 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1612 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1613 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1614 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1615 "zstd compress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1616 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1617 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1618 if out_buffer.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1619 data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1620 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1621 yield data |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1622 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1623 assert out_buffer.pos == 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1624 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1625 # And repeat the loop to collect more data. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1626 continue |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1627 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1628 # If we get here, input is exhausted. End the stream and emit what |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1629 # remains. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1630 while True: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1631 assert out_buffer.pos == 0 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1632 zresult = lib.ZSTD_compressStream2( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1633 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1634 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1635 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1636 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1637 "error ending compression stream: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1638 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1639 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1640 if out_buffer.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1641 data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1642 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1643 yield data |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1644 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1645 if zresult == 0: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1646 break |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1647 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1648 read_from = read_to_iter |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1649 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1650 def frame_progression(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1651 progression = lib.ZSTD_getFrameProgression(self._cctx) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1652 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1653 return progression.ingested, progression.consumed, progression.produced |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1654 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1655 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1656 class FrameParameters(object): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1657 def __init__(self, fparams): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1658 self.content_size = fparams.frameContentSize |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1659 self.window_size = fparams.windowSize |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1660 self.dict_id = fparams.dictID |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1661 self.has_checksum = bool(fparams.checksumFlag) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1662 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1663 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1664 def frame_content_size(data): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1665 data_buffer = ffi.from_buffer(data) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1666 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1667 size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer)) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1668 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1669 if size == lib.ZSTD_CONTENTSIZE_ERROR: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1670 raise ZstdError("error when determining content size") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1671 elif size == lib.ZSTD_CONTENTSIZE_UNKNOWN: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1672 return -1 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1673 else: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1674 return size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1675 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1676 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1677 def frame_header_size(data): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1678 data_buffer = ffi.from_buffer(data) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1679 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1680 zresult = lib.ZSTD_frameHeaderSize(data_buffer, len(data_buffer)) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1681 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1682 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1683 "could not determine frame header size: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1684 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1685 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1686 return zresult |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1687 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1688 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1689 def get_frame_parameters(data): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1690 params = ffi.new("ZSTD_frameHeader *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1691 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1692 data_buffer = ffi.from_buffer(data) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1693 zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer)) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1694 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1695 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1696 "cannot get frame parameters: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1697 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1698 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1699 if zresult: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1700 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1701 "not enough data for frame parameters; need %d bytes" % zresult |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1702 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1703 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1704 return FrameParameters(params[0]) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1705 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1706 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1707 class ZstdCompressionDict(object): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1708 def __init__(self, data, dict_type=DICT_TYPE_AUTO, k=0, d=0): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1709 assert isinstance(data, bytes_type) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1710 self._data = data |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1711 self.k = k |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1712 self.d = d |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1713 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1714 if dict_type not in ( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1715 DICT_TYPE_AUTO, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1716 DICT_TYPE_RAWCONTENT, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1717 DICT_TYPE_FULLDICT, |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1718 ): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1719 raise ValueError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1720 "invalid dictionary load mode: %d; must use " |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1721 "DICT_TYPE_* constants" |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1722 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1723 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1724 self._dict_type = dict_type |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1725 self._cdict = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1726 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1727 def __len__(self): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1728 return len(self._data) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1729 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1730 def dict_id(self): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1731 return int_type(lib.ZDICT_getDictID(self._data, len(self._data))) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1732 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1733 def as_bytes(self): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1734 return self._data |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1735 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1736 def precompute_compress(self, level=0, compression_params=None): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1737 if level and compression_params: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1738 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1739 "must only specify one of level or " "compression_params" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1740 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1741 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1742 if not level and not compression_params: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1743 raise ValueError("must specify one of level or compression_params") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1744 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1745 if level: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1746 cparams = lib.ZSTD_getCParams(level, 0, len(self._data)) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1747 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1748 cparams = ffi.new("ZSTD_compressionParameters") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1749 cparams.chainLog = compression_params.chain_log |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1750 cparams.hashLog = compression_params.hash_log |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1751 cparams.minMatch = compression_params.min_match |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1752 cparams.searchLog = compression_params.search_log |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1753 cparams.strategy = compression_params.compression_strategy |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1754 cparams.targetLength = compression_params.target_length |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1755 cparams.windowLog = compression_params.window_log |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1756 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1757 cdict = lib.ZSTD_createCDict_advanced( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1758 self._data, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1759 len(self._data), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1760 lib.ZSTD_dlm_byRef, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1761 self._dict_type, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1762 cparams, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1763 lib.ZSTD_defaultCMem, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1764 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1765 if cdict == ffi.NULL: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1766 raise ZstdError("unable to precompute dictionary") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1767 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1768 self._cdict = ffi.gc( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1769 cdict, lib.ZSTD_freeCDict, size=lib.ZSTD_sizeof_CDict(cdict) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1770 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1771 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1772 @property |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1773 def _ddict(self): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1774 ddict = lib.ZSTD_createDDict_advanced( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1775 self._data, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1776 len(self._data), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1777 lib.ZSTD_dlm_byRef, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1778 self._dict_type, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1779 lib.ZSTD_defaultCMem, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1780 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1781 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1782 if ddict == ffi.NULL: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1783 raise ZstdError("could not create decompression dict") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1784 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1785 ddict = ffi.gc( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1786 ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1787 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1788 self.__dict__["_ddict"] = ddict |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1789 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1790 return ddict |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1791 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1792 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1793 def train_dictionary( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1794 dict_size, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1795 samples, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1796 k=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1797 d=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1798 notifications=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1799 dict_id=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1800 level=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1801 steps=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1802 threads=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1803 ): |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1804 if not isinstance(samples, list): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1805 raise TypeError("samples must be a list") |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1806 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1807 if threads < 0: |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1808 threads = _cpu_count() |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1809 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1810 total_size = sum(map(len, samples)) |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1811 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1812 samples_buffer = new_nonzero("char[]", total_size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1813 sample_sizes = new_nonzero("size_t[]", len(samples)) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1814 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1815 offset = 0 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1816 for i, sample in enumerate(samples): |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1817 if not isinstance(sample, bytes_type): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1818 raise ValueError("samples must be bytes") |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1819 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1820 l = len(sample) |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1821 ffi.memmove(samples_buffer + offset, sample, l) |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1822 offset += l |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1823 sample_sizes[i] = l |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1824 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1825 dict_data = new_nonzero("char[]", dict_size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1826 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1827 dparams = ffi.new("ZDICT_cover_params_t *")[0] |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1828 dparams.k = k |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1829 dparams.d = d |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1830 dparams.steps = steps |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1831 dparams.nbThreads = threads |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1832 dparams.zParams.notificationLevel = notifications |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1833 dparams.zParams.dictID = dict_id |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1834 dparams.zParams.compressionLevel = level |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1835 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1836 if ( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1837 not dparams.k |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1838 and not dparams.d |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1839 and not dparams.steps |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1840 and not dparams.nbThreads |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1841 and not dparams.zParams.notificationLevel |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1842 and not dparams.zParams.dictID |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1843 and not dparams.zParams.compressionLevel |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1844 ): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1845 zresult = lib.ZDICT_trainFromBuffer( |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1846 ffi.addressof(dict_data), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1847 dict_size, |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1848 ffi.addressof(samples_buffer), |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1849 ffi.addressof(sample_sizes, 0), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1850 len(samples), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1851 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1852 elif dparams.steps or dparams.nbThreads: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1853 zresult = lib.ZDICT_optimizeTrainFromBuffer_cover( |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1854 ffi.addressof(dict_data), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1855 dict_size, |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1856 ffi.addressof(samples_buffer), |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1857 ffi.addressof(sample_sizes, 0), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1858 len(samples), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1859 ffi.addressof(dparams), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1860 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1861 else: |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1862 zresult = lib.ZDICT_trainFromBuffer_cover( |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1863 ffi.addressof(dict_data), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1864 dict_size, |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1865 ffi.addressof(samples_buffer), |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1866 ffi.addressof(sample_sizes, 0), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1867 len(samples), |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1868 dparams, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1869 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1870 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1871 if lib.ZDICT_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1872 msg = ffi.string(lib.ZDICT_getErrorName(zresult)).decode("utf-8") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1873 raise ZstdError("cannot train dict: %s" % msg) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1874 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1875 return ZstdCompressionDict( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1876 ffi.buffer(dict_data, zresult)[:], |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1877 dict_type=DICT_TYPE_FULLDICT, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1878 k=dparams.k, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1879 d=dparams.d, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1880 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1881 |
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
1882 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1883 class ZstdDecompressionObj(object): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1884 def __init__(self, decompressor, write_size): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1885 self._decompressor = decompressor |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1886 self._write_size = write_size |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1887 self._finished = False |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1888 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1889 def decompress(self, data): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1890 if self._finished: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1891 raise ZstdError("cannot use a decompressobj multiple times") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1892 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1893 in_buffer = ffi.new("ZSTD_inBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1894 out_buffer = ffi.new("ZSTD_outBuffer *") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1895 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1896 data_buffer = ffi.from_buffer(data) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1897 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1898 if len(data_buffer) == 0: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1899 return b"" |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1900 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1901 in_buffer.src = data_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1902 in_buffer.size = len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1903 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1904 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1905 dst_buffer = ffi.new("char[]", self._write_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1906 out_buffer.dst = dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1907 out_buffer.size = len(dst_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1908 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1909 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1910 chunks = [] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1911 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1912 while True: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1913 zresult = lib.ZSTD_decompressStream( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1914 self._decompressor._dctx, out_buffer, in_buffer |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1915 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1916 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1917 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1918 "zstd decompressor error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
1919 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1920 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1921 if zresult == 0: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1922 self._finished = True |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1923 self._decompressor = None |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1924 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1925 if out_buffer.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1926 chunks.append(ffi.buffer(out_buffer.dst, out_buffer.pos)[:]) |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1927 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1928 if zresult == 0 or ( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1929 in_buffer.pos == in_buffer.size and out_buffer.pos == 0 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1930 ): |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1931 break |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1932 |
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
1933 out_buffer.pos = 0 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1934 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1935 return b"".join(chunks) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
1936 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1937 def flush(self, length=0): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1938 pass |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1939 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1940 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1941 class ZstdDecompressionReader(object): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1942 def __init__(self, decompressor, source, read_size, read_across_frames): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1943 self._decompressor = decompressor |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1944 self._source = source |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1945 self._read_size = read_size |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1946 self._read_across_frames = bool(read_across_frames) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1947 self._entered = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1948 self._closed = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1949 self._bytes_decompressed = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1950 self._finished_input = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1951 self._finished_output = False |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1952 self._in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1953 # Holds a ref to self._in_buffer.src. |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1954 self._source_buffer = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1955 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1956 def __enter__(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1957 if self._entered: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
1958 raise ValueError("cannot __enter__ multiple times") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1959 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1960 self._entered = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1961 return self |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1962 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1963 def __exit__(self, exc_type, exc_value, exc_tb): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1964 self._entered = False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1965 self._closed = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1966 self._source = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1967 self._decompressor = None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1968 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1969 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1970 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1971 def readable(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1972 return True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1973 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1974 def writable(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1975 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1976 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1977 def seekable(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1978 return True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1979 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1980 def readline(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1981 raise io.UnsupportedOperation() |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1982 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1983 def readlines(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
1984 raise io.UnsupportedOperation() |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1985 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1986 def write(self, data): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1987 raise io.UnsupportedOperation() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1988 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1989 def writelines(self, lines): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1990 raise io.UnsupportedOperation() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1991 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1992 def isatty(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1993 return False |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1994 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1995 def flush(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1996 return None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1997 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1998 def close(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
1999 self._closed = True |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2000 return None |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2001 |
40121
73fef626dae3
zstandard: vendor python-zstandard 0.10.1
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37495
diff
changeset
|
2002 @property |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2003 def closed(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2004 return self._closed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2005 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2006 def tell(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2007 return self._bytes_decompressed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2008 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2009 def readall(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2010 chunks = [] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2011 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2012 while True: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2013 chunk = self.read(1048576) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2014 if not chunk: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2015 break |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2016 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2017 chunks.append(chunk) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2018 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2019 return b"".join(chunks) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2020 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2021 def __iter__(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2022 raise io.UnsupportedOperation() |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2023 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2024 def __next__(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2025 raise io.UnsupportedOperation() |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2026 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2027 next = __next__ |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2028 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2029 def _read_input(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2030 # We have data left over in the input buffer. Use it. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2031 if self._in_buffer.pos < self._in_buffer.size: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2032 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2033 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2034 # All input data exhausted. Nothing to do. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2035 if self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2036 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2037 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2038 # Else populate the input buffer from our source. |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2039 if hasattr(self._source, "read"): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2040 data = self._source.read(self._read_size) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2041 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2042 if not data: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2043 self._finished_input = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2044 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2045 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2046 self._source_buffer = ffi.from_buffer(data) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2047 self._in_buffer.src = self._source_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2048 self._in_buffer.size = len(self._source_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2049 self._in_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2050 else: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2051 self._source_buffer = ffi.from_buffer(self._source) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2052 self._in_buffer.src = self._source_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2053 self._in_buffer.size = len(self._source_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2054 self._in_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2055 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2056 def _decompress_into_buffer(self, out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2057 """Decompress available input into an output buffer. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2058 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2059 Returns True if data in output buffer should be emitted. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2060 """ |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2061 zresult = lib.ZSTD_decompressStream( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2062 self._decompressor._dctx, out_buffer, self._in_buffer |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2063 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2064 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2065 if self._in_buffer.pos == self._in_buffer.size: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2066 self._in_buffer.src = ffi.NULL |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2067 self._in_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2068 self._in_buffer.size = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2069 self._source_buffer = None |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2070 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2071 if not hasattr(self._source, "read"): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2072 self._finished_input = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2073 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2074 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2075 raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult)) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2076 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2077 # Emit data if there is data AND either: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2078 # a) output buffer is full (read amount is satisfied) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2079 # b) we're at end of a frame and not in frame spanning mode |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2080 return out_buffer.pos and ( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2081 out_buffer.pos == out_buffer.size |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2082 or zresult == 0 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2083 and not self._read_across_frames |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2084 ) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2085 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2086 def read(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2087 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2088 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2089 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2090 if size < -1: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2091 raise ValueError("cannot read negative amounts less than -1") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2092 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2093 if size == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2094 # This is recursive. But it gets the job done. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2095 return self.readall() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2096 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2097 if self._finished_output or size == 0: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2098 return b"" |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2099 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2100 # We /could/ call into readinto() here. But that introduces more |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2101 # overhead. |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2102 dst_buffer = ffi.new("char[]", size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2103 out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2104 out_buffer.dst = dst_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2105 out_buffer.size = size |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2106 out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2107 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2108 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2109 if self._decompress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2110 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2111 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2112 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2113 while not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2114 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2115 if self._decompress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2116 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2117 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2118 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2119 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2120 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2121 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2122 def readinto(self, b): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2123 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2124 raise ValueError("stream is closed") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2125 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2126 if self._finished_output: |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2127 return 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2128 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2129 # TODO use writable=True once we require CFFI >= 1.12. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2130 dest_buffer = ffi.from_buffer(b) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2131 ffi.memmove(b, b"", 0) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2132 out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2133 out_buffer.dst = dest_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2134 out_buffer.size = len(dest_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2135 out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2136 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2137 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2138 if self._decompress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2139 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2140 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2141 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2142 while not self._finished_input: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2143 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2144 if self._decompress_into_buffer(out_buffer): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2145 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2146 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2147 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2148 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2149 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2150 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2151 def read1(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2152 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2153 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2154 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2155 if size < -1: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2156 raise ValueError("cannot read negative amounts less than -1") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2157 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2158 if self._finished_output or size == 0: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2159 return b"" |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2160 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2161 # -1 returns arbitrary number of bytes. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2162 if size == -1: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2163 size = DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2164 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2165 dst_buffer = ffi.new("char[]", size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2166 out_buffer = ffi.new("ZSTD_outBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2167 out_buffer.dst = dst_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2168 out_buffer.size = size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2169 out_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2170 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2171 # read1() dictates that we can perform at most 1 call to underlying |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2172 # stream to get input. However, we can't satisfy this restriction with |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2173 # decompression because not all input generates output. So we allow |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2174 # multiple read(). But unlike read(), we stop once we have any output. |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2175 while not self._finished_input: |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2176 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2177 self._decompress_into_buffer(out_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2178 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2179 if out_buffer.pos: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2180 break |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2181 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2182 self._bytes_decompressed += out_buffer.pos |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2183 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2184 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2185 def readinto1(self, b): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2186 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2187 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2188 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2189 if self._finished_output: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2190 return 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2191 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2192 # TODO use writable=True once we require CFFI >= 1.12. |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2193 dest_buffer = ffi.from_buffer(b) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2194 ffi.memmove(b, b"", 0) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2195 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2196 out_buffer = ffi.new("ZSTD_outBuffer *") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2197 out_buffer.dst = dest_buffer |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2198 out_buffer.size = len(dest_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2199 out_buffer.pos = 0 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2200 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2201 while not self._finished_input and not self._finished_output: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2202 self._read_input() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2203 self._decompress_into_buffer(out_buffer) |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2204 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2205 if out_buffer.pos: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2206 break |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2207 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2208 self._bytes_decompressed += out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2209 return out_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2210 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2211 def seek(self, pos, whence=os.SEEK_SET): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2212 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2213 raise ValueError("stream is closed") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2214 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2215 read_amount = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2216 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2217 if whence == os.SEEK_SET: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2218 if pos < 0: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2219 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2220 "cannot seek to negative position with SEEK_SET" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2221 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2222 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2223 if pos < self._bytes_decompressed: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2224 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2225 "cannot seek zstd decompression stream " "backwards" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2226 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2227 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2228 read_amount = pos - self._bytes_decompressed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2229 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2230 elif whence == os.SEEK_CUR: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2231 if pos < 0: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2232 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2233 "cannot seek zstd decompression stream " "backwards" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2234 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2235 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2236 read_amount = pos |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2237 elif whence == os.SEEK_END: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2238 raise ValueError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2239 "zstd decompression streams cannot be seeked " "with SEEK_END" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2240 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2241 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2242 while read_amount: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2243 result = self.read( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2244 min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2245 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2246 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2247 if not result: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2248 break |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2249 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2250 read_amount -= len(result) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2251 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2252 return self._bytes_decompressed |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2253 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2254 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2255 class ZstdDecompressionWriter(object): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2256 def __init__(self, decompressor, writer, write_size, write_return_read): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2257 decompressor._ensure_dctx() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2258 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2259 self._decompressor = decompressor |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2260 self._writer = writer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2261 self._write_size = write_size |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2262 self._write_return_read = bool(write_return_read) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2263 self._entered = False |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2264 self._closed = False |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2265 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2266 def __enter__(self): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2267 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2268 raise ValueError("stream is closed") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2269 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2270 if self._entered: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2271 raise ZstdError("cannot __enter__ multiple times") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2272 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2273 self._entered = True |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2274 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2275 return self |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2276 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2277 def __exit__(self, exc_type, exc_value, exc_tb): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2278 self._entered = False |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2279 self.close() |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2280 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2281 def memory_size(self): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2282 return lib.ZSTD_sizeof_DCtx(self._decompressor._dctx) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2283 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2284 def close(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2285 if self._closed: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2286 return |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2287 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2288 try: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2289 self.flush() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2290 finally: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2291 self._closed = True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2292 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2293 f = getattr(self._writer, "close", None) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2294 if f: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2295 f() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2296 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2297 @property |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2298 def closed(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2299 return self._closed |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2300 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2301 def fileno(self): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2302 f = getattr(self._writer, "fileno", None) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2303 if f: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2304 return f() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2305 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2306 raise OSError("fileno not available on underlying writer") |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2307 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2308 def flush(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2309 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2310 raise ValueError("stream is closed") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2311 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2312 f = getattr(self._writer, "flush", None) |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2313 if f: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2314 return f() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2315 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2316 def isatty(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2317 return False |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2318 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2319 def readable(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2320 return False |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2321 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2322 def readline(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2323 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2324 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2325 def readlines(self, hint=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2326 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2327 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2328 def seek(self, offset, whence=None): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2329 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2330 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2331 def seekable(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2332 return False |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2333 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2334 def tell(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2335 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2336 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2337 def truncate(self, size=None): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2338 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2339 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2340 def writable(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2341 return True |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2342 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2343 def writelines(self, lines): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2344 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2345 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2346 def read(self, size=-1): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2347 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2348 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2349 def readall(self): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2350 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2351 |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2352 def readinto(self, b): |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2353 raise io.UnsupportedOperation() |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2354 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2355 def write(self, data): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2356 if self._closed: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2357 raise ValueError("stream is closed") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2358 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2359 total_write = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2360 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2361 in_buffer = ffi.new("ZSTD_inBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2362 out_buffer = ffi.new("ZSTD_outBuffer *") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2363 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2364 data_buffer = ffi.from_buffer(data) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2365 in_buffer.src = data_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2366 in_buffer.size = len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2367 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2368 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2369 dst_buffer = ffi.new("char[]", self._write_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2370 out_buffer.dst = dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2371 out_buffer.size = len(dst_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2372 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2373 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2374 dctx = self._decompressor._dctx |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
2375 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2376 while in_buffer.pos < in_buffer.size: |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2377 zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2378 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2379 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2380 "zstd decompress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2381 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2382 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2383 if out_buffer.pos: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2384 self._writer.write( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2385 ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2386 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2387 total_write += out_buffer.pos |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2388 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2389 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2390 if self._write_return_read: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2391 return in_buffer.pos |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2392 else: |
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2393 return total_write |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2394 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2395 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2396 class ZstdDecompressor(object): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2397 def __init__(self, dict_data=None, max_window_size=0, format=FORMAT_ZSTD1): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2398 self._dict_data = dict_data |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2399 self._max_window_size = max_window_size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2400 self._format = format |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2401 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2402 dctx = lib.ZSTD_createDCtx() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2403 if dctx == ffi.NULL: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2404 raise MemoryError() |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2405 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2406 self._dctx = dctx |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2407 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2408 # Defer setting up garbage collection until full state is loaded so |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2409 # the memory size is more accurate. |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2410 try: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2411 self._ensure_dctx() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2412 finally: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2413 self._dctx = ffi.gc( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2414 dctx, lib.ZSTD_freeDCtx, size=lib.ZSTD_sizeof_DCtx(dctx) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2415 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2416 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2417 def memory_size(self): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2418 return lib.ZSTD_sizeof_DCtx(self._dctx) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2419 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2420 def decompress(self, data, max_output_size=0): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2421 self._ensure_dctx() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2422 |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2423 data_buffer = ffi.from_buffer(data) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2424 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2425 output_size = lib.ZSTD_getFrameContentSize( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2426 data_buffer, len(data_buffer) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2427 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2428 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2429 if output_size == lib.ZSTD_CONTENTSIZE_ERROR: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2430 raise ZstdError("error determining content size from frame header") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2431 elif output_size == 0: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2432 return b"" |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2433 elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN: |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2434 if not max_output_size: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2435 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2436 "could not determine content size in frame header" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2437 ) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2438 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2439 result_buffer = ffi.new("char[]", max_output_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2440 result_size = max_output_size |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2441 output_size = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2442 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2443 result_buffer = ffi.new("char[]", output_size) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2444 result_size = output_size |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2445 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2446 out_buffer = ffi.new("ZSTD_outBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2447 out_buffer.dst = result_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2448 out_buffer.size = result_size |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2449 out_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2450 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2451 in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2452 in_buffer.src = data_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2453 in_buffer.size = len(data_buffer) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2454 in_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2455 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2456 zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2457 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2458 raise ZstdError("decompression error: %s" % _zstd_error(zresult)) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2459 elif zresult: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2460 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2461 "decompression error: did not decompress full frame" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2462 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2463 elif output_size and out_buffer.pos != output_size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2464 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2465 "decompression error: decompressed %d bytes; expected %d" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2466 % (zresult, output_size) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2467 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2468 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2469 return ffi.buffer(result_buffer, out_buffer.pos)[:] |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2470 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2471 def stream_reader( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2472 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2473 source, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2474 read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2475 read_across_frames=False, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2476 ): |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2477 self._ensure_dctx() |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2478 return ZstdDecompressionReader( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2479 self, source, read_size, read_across_frames |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2480 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2481 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2482 def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE): |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2483 if write_size < 1: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2484 raise ValueError("write_size must be positive") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2485 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2486 self._ensure_dctx() |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2487 return ZstdDecompressionObj(self, write_size=write_size) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2488 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2489 def read_to_iter( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2490 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2491 reader, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2492 read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2493 write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2494 skip_bytes=0, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2495 ): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2496 if skip_bytes >= read_size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2497 raise ValueError("skip_bytes must be smaller than read_size") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2498 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2499 if hasattr(reader, "read"): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2500 have_read = True |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2501 elif hasattr(reader, "__getitem__"): |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2502 have_read = False |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2503 buffer_offset = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2504 size = len(reader) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2505 else: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2506 raise ValueError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2507 "must pass an object with a read() method or " |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2508 "conforms to buffer protocol" |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2509 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2510 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2511 if skip_bytes: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2512 if have_read: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2513 reader.read(skip_bytes) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2514 else: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2515 if skip_bytes > size: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2516 raise ValueError("skip_bytes larger than first input chunk") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2517 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2518 buffer_offset = skip_bytes |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2519 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2520 self._ensure_dctx() |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2521 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2522 in_buffer = ffi.new("ZSTD_inBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2523 out_buffer = ffi.new("ZSTD_outBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2524 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2525 dst_buffer = ffi.new("char[]", write_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2526 out_buffer.dst = dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2527 out_buffer.size = len(dst_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2528 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2529 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2530 while True: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2531 assert out_buffer.pos == 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2532 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2533 if have_read: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2534 read_result = reader.read(read_size) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2535 else: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2536 remaining = size - buffer_offset |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2537 slice_size = min(remaining, read_size) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2538 read_result = reader[buffer_offset : buffer_offset + slice_size] |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2539 buffer_offset += slice_size |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2540 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2541 # No new input. Break out of read loop. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2542 if not read_result: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2543 break |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2544 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2545 # Feed all read data into decompressor and emit output until |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2546 # exhausted. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2547 read_buffer = ffi.from_buffer(read_result) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2548 in_buffer.src = read_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2549 in_buffer.size = len(read_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2550 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2551 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2552 while in_buffer.pos < in_buffer.size: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2553 assert out_buffer.pos == 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2554 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2555 zresult = lib.ZSTD_decompressStream( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2556 self._dctx, out_buffer, in_buffer |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2557 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2558 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2559 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2560 "zstd decompress error: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2561 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2562 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2563 if out_buffer.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2564 data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2565 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2566 yield data |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2567 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2568 if zresult == 0: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2569 return |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2570 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2571 # Repeat loop to collect more input data. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2572 continue |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2573 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2574 # If we get here, input is exhausted. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2575 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2576 read_from = read_to_iter |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2577 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2578 def stream_writer( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2579 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2580 writer, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2581 write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2582 write_return_read=False, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2583 ): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2584 if not hasattr(writer, "write"): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2585 raise ValueError("must pass an object with a write() method") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2586 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2587 return ZstdDecompressionWriter( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2588 self, writer, write_size, write_return_read |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2589 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2590 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2591 write_to = stream_writer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2592 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2593 def copy_stream( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2594 self, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2595 ifh, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2596 ofh, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2597 read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2598 write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE, |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2599 ): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2600 if not hasattr(ifh, "read"): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2601 raise ValueError("first argument must have a read() method") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2602 if not hasattr(ofh, "write"): |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2603 raise ValueError("second argument must have a write() method") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2604 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2605 self._ensure_dctx() |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2606 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2607 in_buffer = ffi.new("ZSTD_inBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2608 out_buffer = ffi.new("ZSTD_outBuffer *") |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2609 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2610 dst_buffer = ffi.new("char[]", write_size) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2611 out_buffer.dst = dst_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2612 out_buffer.size = write_size |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2613 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2614 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2615 total_read, total_write = 0, 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2616 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2617 # Read all available input. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2618 while True: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2619 data = ifh.read(read_size) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2620 if not data: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2621 break |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2622 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2623 data_buffer = ffi.from_buffer(data) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2624 total_read += len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2625 in_buffer.src = data_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2626 in_buffer.size = len(data_buffer) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2627 in_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2628 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2629 # Flush all read data to output. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2630 while in_buffer.pos < in_buffer.size: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2631 zresult = lib.ZSTD_decompressStream( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2632 self._dctx, out_buffer, in_buffer |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2633 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2634 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2635 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2636 "zstd decompressor error: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2637 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2638 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2639 if out_buffer.pos: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2640 ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2641 total_write += out_buffer.pos |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2642 out_buffer.pos = 0 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2643 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2644 # Continue loop to keep reading. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2645 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2646 return total_read, total_write |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2647 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2648 def decompress_content_dict_chain(self, frames): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2649 if not isinstance(frames, list): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2650 raise TypeError("argument must be a list") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2651 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2652 if not frames: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2653 raise ValueError("empty input chain") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2654 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2655 # First chunk should not be using a dictionary. We handle it specially. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2656 chunk = frames[0] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2657 if not isinstance(chunk, bytes_type): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2658 raise ValueError("chunk 0 must be bytes") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2659 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2660 # All chunks should be zstd frames and should have content size set. |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2661 chunk_buffer = ffi.from_buffer(chunk) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2662 params = ffi.new("ZSTD_frameHeader *") |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2663 zresult = lib.ZSTD_getFrameHeader( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2664 params, chunk_buffer, len(chunk_buffer) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2665 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2666 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2667 raise ValueError("chunk 0 is not a valid zstd frame") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2668 elif zresult: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2669 raise ValueError("chunk 0 is too small to contain a zstd frame") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2670 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2671 if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2672 raise ValueError("chunk 0 missing content size in frame") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2673 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2674 self._ensure_dctx(load_dict=False) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2675 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2676 last_buffer = ffi.new("char[]", params.frameContentSize) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2677 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2678 out_buffer = ffi.new("ZSTD_outBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2679 out_buffer.dst = last_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2680 out_buffer.size = len(last_buffer) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2681 out_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2682 |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2683 in_buffer = ffi.new("ZSTD_inBuffer *") |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2684 in_buffer.src = chunk_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2685 in_buffer.size = len(chunk_buffer) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2686 in_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2687 |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2688 zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2689 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2690 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2691 "could not decompress chunk 0: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2692 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2693 elif zresult: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2694 raise ZstdError("chunk 0 did not decompress full frame") |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2695 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2696 # Special case of chain length of 1 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2697 if len(frames) == 1: |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2698 return ffi.buffer(last_buffer, len(last_buffer))[:] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2699 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2700 i = 1 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2701 while i < len(frames): |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2702 chunk = frames[i] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2703 if not isinstance(chunk, bytes_type): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2704 raise ValueError("chunk %d must be bytes" % i) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2705 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2706 chunk_buffer = ffi.from_buffer(chunk) |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2707 zresult = lib.ZSTD_getFrameHeader( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2708 params, chunk_buffer, len(chunk_buffer) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2709 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2710 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2711 raise ValueError("chunk %d is not a valid zstd frame" % i) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2712 elif zresult: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2713 raise ValueError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2714 "chunk %d is too small to contain a zstd frame" % i |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2715 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2716 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2717 if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2718 raise ValueError("chunk %d missing content size in frame" % i) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2719 |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2720 dest_buffer = ffi.new("char[]", params.frameContentSize) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2721 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2722 out_buffer.dst = dest_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2723 out_buffer.size = len(dest_buffer) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2724 out_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2725 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2726 in_buffer.src = chunk_buffer |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2727 in_buffer.size = len(chunk_buffer) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2728 in_buffer.pos = 0 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2729 |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2730 zresult = lib.ZSTD_decompressStream( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2731 self._dctx, out_buffer, in_buffer |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2732 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2733 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2734 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2735 "could not decompress chunk %d: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2736 ) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2737 elif zresult: |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2738 raise ZstdError("chunk %d did not decompress full frame" % i) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2739 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2740 last_buffer = dest_buffer |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2741 i += 1 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2742 |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2743 return ffi.buffer(last_buffer, len(last_buffer))[:] |
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2744 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2745 def _ensure_dctx(self, load_dict=True): |
42070
675775c33ab6
zstandard: vendor python-zstandard 0.11
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40121
diff
changeset
|
2746 lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only) |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2747 |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2748 if self._max_window_size: |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2749 zresult = lib.ZSTD_DCtx_setMaxWindowSize( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2750 self._dctx, self._max_window_size |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2751 ) |
31796
e0dc40530c5a
zstd: vendor python-zstandard 0.8.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30895
diff
changeset
|
2752 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2753 raise ZstdError( |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2754 "unable to set max window size: %s" % _zstd_error(zresult) |
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2755 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2756 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2757 zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2758 if lib.ZSTD_isError(zresult): |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2759 raise ZstdError( |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2760 "unable to set decoding format: %s" % _zstd_error(zresult) |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2761 ) |
30895
c32454d69b85
zstd: vendor python-zstandard 0.7.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30435
diff
changeset
|
2762 |
37495
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2763 if self._dict_data and load_dict: |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2764 zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict) |
b1fb341d8a61
zstandard: vendor python-zstandard 0.9.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31796
diff
changeset
|
2765 if lib.ZSTD_isError(zresult): |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2766 raise ZstdError( |
44147
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2767 "unable to reference prepared dictionary: %s" |
5e84a96d865b
python-zstandard: blacken at 80 characters
Gregory Szorc <gregory.szorc@gmail.com>
parents:
43994
diff
changeset
|
2768 % _zstd_error(zresult) |
43994
de7838053207
zstandard: vendor python-zstandard 0.13.0
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42937
diff
changeset
|
2769 ) |