33 #else |
33 #else |
34 # define ZSTDLIB_API ZSTDLIB_VISIBILITY |
34 # define ZSTDLIB_API ZSTDLIB_VISIBILITY |
35 #endif |
35 #endif |
36 |
36 |
37 |
37 |
38 /******************************************************************************************************* |
38 /******************************************************************************* |
39 Introduction |
39 Introduction |
40 |
40 |
41 zstd, short for Zstandard, is a fast lossless compression algorithm, |
41 zstd, short for Zstandard, is a fast lossless compression algorithm, targeting |
42 targeting real-time compression scenarios at zlib-level and better compression ratios. |
42 real-time compression scenarios at zlib-level and better compression ratios. |
43 The zstd compression library provides in-memory compression and decompression functions. |
43 The zstd compression library provides in-memory compression and decompression |
44 The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22. |
44 functions. |
45 Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory. |
45 |
|
46 The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), |
|
47 which is currently 22. Levels >= 20, labeled `--ultra`, should be used with |
|
48 caution, as they require more memory. The library also offers negative |
|
49 compression levels, which extend the range of speed vs. ratio preferences. |
|
50 The lower the level, the faster the speed (at the cost of compression). |
|
51 |
46 Compression can be done in: |
52 Compression can be done in: |
47 - a single step (described as Simple API) |
53 - a single step (described as Simple API) |
48 - a single step, reusing a context (described as Explicit context) |
54 - a single step, reusing a context (described as Explicit context) |
49 - unbounded multiple steps (described as Streaming compression) |
55 - unbounded multiple steps (described as Streaming compression) |
50 The compression ratio achievable on small data can be highly improved using a dictionary in: |
56 |
|
57 The compression ratio achievable on small data can be highly improved using |
|
58 a dictionary. Dictionary compression can be performed in: |
51 - a single step (described as Simple dictionary API) |
59 - a single step (described as Simple dictionary API) |
52 - a single step, reusing a dictionary (described as Bulk-processing dictionary API) |
60 - a single step, reusing a dictionary (described as Bulk-processing |
53 |
61 dictionary API) |
54 Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. |
62 |
55 Advanced experimental APIs shall never be used with a dynamic library. |
63 Advanced experimental functions can be accessed using |
56 They are not "stable", their definition may change in the future. Only static linking is allowed. |
64 `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. |
57 *********************************************************************************************************/ |
65 |
|
66 Advanced experimental APIs should never be used with a dynamically-linked |
|
67 library. They are not "stable"; their definitions or signatures may change in |
|
68 the future. Only static linking is allowed. |
|
69 *******************************************************************************/ |
58 |
70 |
59 /*------ Version ------*/ |
71 /*------ Version ------*/ |
60 #define ZSTD_VERSION_MAJOR 1 |
72 #define ZSTD_VERSION_MAJOR 1 |
61 #define ZSTD_VERSION_MINOR 3 |
73 #define ZSTD_VERSION_MINOR 3 |
62 #define ZSTD_VERSION_RELEASE 4 |
74 #define ZSTD_VERSION_RELEASE 6 |
63 |
75 |
64 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) |
76 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) |
65 ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */ |
77 ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */ |
66 |
78 |
67 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE |
79 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE |
68 #define ZSTD_QUOTE(str) #str |
80 #define ZSTD_QUOTE(str) #str |
69 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) |
81 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) |
70 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) |
82 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) |
71 ZSTDLIB_API const char* ZSTD_versionString(void); /* added in v1.3.0 */ |
83 ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0+ */ |
72 |
84 |
|
85 /*************************************** |
|
86 * Default constant |
|
87 ***************************************/ |
|
88 #ifndef ZSTD_CLEVEL_DEFAULT |
|
89 # define ZSTD_CLEVEL_DEFAULT 3 |
|
90 #endif |
73 |
91 |
74 /*************************************** |
92 /*************************************** |
75 * Simple API |
93 * Simple API |
76 ***************************************/ |
94 ***************************************/ |
77 /*! ZSTD_compress() : |
95 /*! ZSTD_compress() : |
94 |
112 |
95 /*! ZSTD_getFrameContentSize() : added in v1.3.0 |
113 /*! ZSTD_getFrameContentSize() : added in v1.3.0 |
96 * `src` should point to the start of a ZSTD encoded frame. |
114 * `src` should point to the start of a ZSTD encoded frame. |
97 * `srcSize` must be at least as large as the frame header. |
115 * `srcSize` must be at least as large as the frame header. |
98 * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. |
116 * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. |
99 * @return : - decompressed size of the frame in `src`, if known |
117 * @return : - decompressed size of `src` frame content, if known |
100 * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined |
118 * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined |
101 * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) |
119 * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) |
102 * note 1 : a 0 return value means the frame is valid but "empty". |
120 * note 1 : a 0 return value means the frame is valid but "empty". |
103 * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. |
121 * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. |
104 * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. |
122 * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. |
105 * In which case, it's necessary to use streaming mode to decompress data. |
123 * In which case, it's necessary to use streaming mode to decompress data. |
106 * Optionally, application can rely on some implicit limit, |
124 * Optionally, application can rely on some implicit limit, |
107 * as ZSTD_decompress() only needs an upper bound of decompressed size. |
125 * as ZSTD_decompress() only needs an upper bound of decompressed size. |
108 * (For example, data could be necessarily cut into blocks <= 16 KB). |
126 * (For example, data could be necessarily cut into blocks <= 16 KB). |
109 * note 3 : decompressed size is always present when compression is done with ZSTD_compress() |
127 * note 3 : decompressed size is always present when compression is completed using single-pass functions, |
|
128 * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). |
110 * note 4 : decompressed size can be very large (64-bits value), |
129 * note 4 : decompressed size can be very large (64-bits value), |
111 * potentially larger than what local system can handle as a single memory segment. |
130 * potentially larger than what local system can handle as a single memory segment. |
112 * In which case, it's necessary to use streaming mode to decompress data. |
131 * In which case, it's necessary to use streaming mode to decompress data. |
113 * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. |
132 * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. |
114 * Always ensure return value fits within application's authorized limits. |
133 * Always ensure return value fits within application's authorized limits. |
121 /*! ZSTD_getDecompressedSize() : |
140 /*! ZSTD_getDecompressedSize() : |
122 * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). |
141 * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). |
123 * Both functions work the same way, but ZSTD_getDecompressedSize() blends |
142 * Both functions work the same way, but ZSTD_getDecompressedSize() blends |
124 * "empty", "unknown" and "error" results to the same return value (0), |
143 * "empty", "unknown" and "error" results to the same return value (0), |
125 * while ZSTD_getFrameContentSize() gives them separate return values. |
144 * while ZSTD_getFrameContentSize() gives them separate return values. |
126 * `src` is the start of a zstd compressed frame. |
145 * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ |
127 * @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */ |
|
128 ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); |
146 ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); |
129 |
147 |
130 |
148 |
131 /*====== Helper functions ======*/ |
149 /*====== Helper functions ======*/ |
132 #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ |
150 #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ |
203 |
221 |
204 /*! ZSTD_createCDict() : |
222 /*! ZSTD_createCDict() : |
205 * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. |
223 * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. |
206 * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. |
224 * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. |
207 * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. |
225 * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. |
208 * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */ |
226 * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict |
|
227 * Note : A ZSTD_CDict can be created with an empty dictionary, but it is inefficient for small data. */ |
209 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, |
228 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, |
210 int compressionLevel); |
229 int compressionLevel); |
211 |
230 |
212 /*! ZSTD_freeCDict() : |
231 /*! ZSTD_freeCDict() : |
213 * Function frees memory allocated by ZSTD_createCDict(). */ |
232 * Function frees memory allocated by ZSTD_createCDict(). */ |
215 |
234 |
216 /*! ZSTD_compress_usingCDict() : |
235 /*! ZSTD_compress_usingCDict() : |
217 * Compression using a digested Dictionary. |
236 * Compression using a digested Dictionary. |
218 * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. |
237 * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. |
219 * Note that compression level is decided during dictionary creation. |
238 * Note that compression level is decided during dictionary creation. |
220 * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ |
239 * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) |
|
240 * Note : ZSTD_compress_usingCDict() can be used with a ZSTD_CDict created from an empty dictionary. |
|
241 * But it is inefficient for small data, and it is recommended to use ZSTD_compressCCtx(). */ |
221 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, |
242 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, |
222 void* dst, size_t dstCapacity, |
243 void* dst, size_t dstCapacity, |
223 const void* src, size_t srcSize, |
244 const void* src, size_t srcSize, |
224 const ZSTD_CDict* cdict); |
245 const ZSTD_CDict* cdict); |
225 |
246 |
270 * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. |
291 * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. |
271 * It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, |
292 * It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, |
272 * since it will play nicer with system's memory, by re-using already allocated memory. |
293 * since it will play nicer with system's memory, by re-using already allocated memory. |
273 * Use one separate ZSTD_CStream per thread for parallel execution. |
294 * Use one separate ZSTD_CStream per thread for parallel execution. |
274 * |
295 * |
275 * Start a new compression by initializing ZSTD_CStream. |
296 * Start a new compression by initializing ZSTD_CStream context. |
276 * Use ZSTD_initCStream() to start a new compression operation. |
297 * Use ZSTD_initCStream() to start a new compression operation. |
277 * Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section) |
298 * Use variants ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for streaming with dictionary (experimental section) |
278 * |
299 * |
279 * Use ZSTD_compressStream() repetitively to consume input stream. |
300 * Use ZSTD_compressStream() as many times as necessary to consume input stream. |
280 * The function will automatically update both `pos` fields. |
301 * The function will automatically update both `pos` fields within `input` and `output`. |
281 * Note that it may not consume the entire input, in which case `pos < size`, |
302 * Note that the function may not consume the entire input, |
282 * and it's up to the caller to present again remaining data. |
303 * for example, because the output buffer is already full, |
|
304 * in which case `input.pos < input.size`. |
|
305 * The caller must check if input has been entirely consumed. |
|
306 * If not, the caller must make some room to receive more compressed data, |
|
307 * typically by emptying output buffer, or allocating a new output buffer, |
|
308 * and then present again remaining input data. |
283 * @return : a size hint, preferred nb of bytes to use as input for next function call |
309 * @return : a size hint, preferred nb of bytes to use as input for next function call |
284 * or an error code, which can be tested using ZSTD_isError(). |
310 * or an error code, which can be tested using ZSTD_isError(). |
285 * Note 1 : it's just a hint, to help latency a little, any other value will work fine. |
311 * Note 1 : it's just a hint, to help latency a little, any other value will work fine. |
286 * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() |
312 * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() |
287 * |
313 * |
288 * At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream(). |
314 * At any moment, it's possible to flush whatever data might remain stuck within internal buffer, |
289 * `output->pos` will be updated. |
315 * using ZSTD_flushStream(). `output->pos` will be updated. |
290 * Note that some content might still be left within internal buffer if `output->size` is too small. |
316 * Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0). |
291 * @return : nb of bytes still present within internal buffer (0 if it's empty) |
317 * In which case, make some room to receive more compressed data, and call again ZSTD_flushStream(). |
|
318 * @return : 0 if internal buffers are entirely flushed, |
|
319 * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), |
292 * or an error code, which can be tested using ZSTD_isError(). |
320 * or an error code, which can be tested using ZSTD_isError(). |
293 * |
321 * |
294 * ZSTD_endStream() instructs to finish a frame. |
322 * ZSTD_endStream() instructs to finish a frame. |
295 * It will perform a flush and write frame epilogue. |
323 * It will perform a flush and write frame epilogue. |
296 * The epilogue is required for decoders to consider a frame completed. |
324 * The epilogue is required for decoders to consider a frame completed. |
297 * ZSTD_endStream() may not be able to flush full data if `output->size` is too small. |
325 * flush() operation is the same, and follows same rules as ZSTD_flushStream(). |
298 * In which case, call again ZSTD_endStream() to complete the flush. |
|
299 * @return : 0 if frame fully completed and fully flushed, |
326 * @return : 0 if frame fully completed and fully flushed, |
300 or >0 if some data is still present within internal buffer |
327 * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), |
301 (value is minimum size estimation for remaining data to flush, but it could be more) |
|
302 * or an error code, which can be tested using ZSTD_isError(). |
328 * or an error code, which can be tested using ZSTD_isError(). |
303 * |
329 * |
304 * *******************************************************************/ |
330 * *******************************************************************/ |
305 |
331 |
306 typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ |
332 typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ |
307 /* Continue to distinguish them for compatibility with versions <= v1.2.0 */ |
333 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ |
308 /*===== ZSTD_CStream management functions =====*/ |
334 /*===== ZSTD_CStream management functions =====*/ |
309 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); |
335 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); |
310 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); |
336 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); |
311 |
337 |
312 /*===== Streaming compression functions =====*/ |
338 /*===== Streaming compression functions =====*/ |
357 |
383 |
358 #endif /* ZSTD_H_235446 */ |
384 #endif /* ZSTD_H_235446 */ |
359 |
385 |
360 |
386 |
361 |
387 |
|
388 |
|
389 #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) |
|
390 #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY |
|
391 |
362 /**************************************************************************************** |
392 /**************************************************************************************** |
363 * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS |
393 * ADVANCED AND EXPERIMENTAL FUNCTIONS |
|
394 **************************************************************************************** |
364 * The definitions in this section are considered experimental. |
395 * The definitions in this section are considered experimental. |
365 * They should never be used with a dynamic library, as prototypes may change in the future. |
396 * They should never be used with a dynamic library, as prototypes may change in the future. |
366 * They are provided for advanced scenarios. |
397 * They are provided for advanced scenarios. |
367 * Use them only in association with static linking. |
398 * Use them only in association with static linking. |
368 * ***************************************************************************************/ |
399 * ***************************************************************************************/ |
369 |
400 |
370 #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) |
401 ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ |
371 #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY |
402 |
372 |
403 /* --- Constants ---*/ |
373 /* --- Constants ---*/ |
404 #define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8+ */ |
374 #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ |
405 #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */ |
375 #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U |
406 #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U |
376 #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* >= v0.7.0 */ |
407 |
|
408 #define ZSTD_BLOCKSIZELOG_MAX 17 |
|
409 #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */ |
377 |
410 |
378 #define ZSTD_WINDOWLOG_MAX_32 30 |
411 #define ZSTD_WINDOWLOG_MAX_32 30 |
379 #define ZSTD_WINDOWLOG_MAX_64 31 |
412 #define ZSTD_WINDOWLOG_MAX_64 31 |
380 #define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) |
413 #define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) |
381 #define ZSTD_WINDOWLOG_MIN 10 |
414 #define ZSTD_WINDOWLOG_MIN 10 |
388 #define ZSTD_HASHLOG3_MAX 17 |
421 #define ZSTD_HASHLOG3_MAX 17 |
389 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) |
422 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) |
390 #define ZSTD_SEARCHLOG_MIN 1 |
423 #define ZSTD_SEARCHLOG_MIN 1 |
391 #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ |
424 #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ |
392 #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ |
425 #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ |
393 #define ZSTD_TARGETLENGTH_MIN 1 /* only used by btopt, btultra and btfast */ |
426 #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX |
|
427 #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ |
|
428 #define ZSTD_LDM_MINMATCH_MAX 4096 |
394 #define ZSTD_LDM_MINMATCH_MIN 4 |
429 #define ZSTD_LDM_MINMATCH_MIN 4 |
395 #define ZSTD_LDM_MINMATCH_MAX 4096 |
|
396 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8 |
430 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8 |
397 |
431 |
398 #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */ |
432 #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */ |
399 #define ZSTD_FRAMEHEADERSIZE_MIN 6 |
433 #define ZSTD_FRAMEHEADERSIZE_MIN 6 |
400 #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ |
434 #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ |
402 static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; |
436 static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; |
403 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; |
437 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; |
404 static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ |
438 static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ |
405 |
439 |
406 |
440 |
407 /*--- Advanced types ---*/ |
441 |
|
442 /* --- Advanced types --- */ |
408 typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, |
443 typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, |
409 ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ |
444 ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ |
410 |
445 |
411 typedef struct { |
446 typedef struct { |
412 unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ |
447 unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ |
478 * read each contained frame header. This is fast as most of the data is skipped, |
513 * read each contained frame header. This is fast as most of the data is skipped, |
479 * however it does mean that all frame data must be present and valid. */ |
514 * however it does mean that all frame data must be present and valid. */ |
480 ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); |
515 ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); |
481 |
516 |
482 /*! ZSTD_frameHeaderSize() : |
517 /*! ZSTD_frameHeaderSize() : |
483 * `src` should point to the start of a ZSTD frame |
518 * srcSize must be >= ZSTD_frameHeaderSize_prefix. |
484 * `srcSize` must be >= ZSTD_frameHeaderSize_prefix. |
519 * @return : size of the Frame Header, |
485 * @return : size of the Frame Header */ |
520 * or an error code (if srcSize is too small) */ |
486 ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); |
521 ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); |
487 |
522 |
488 |
523 |
489 /*************************************** |
524 /*************************************** |
490 * Memory management |
525 * Memory management |
709 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ |
744 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ |
710 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ |
745 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ |
711 |
746 |
712 /*! ZSTD_resetCStream() : |
747 /*! ZSTD_resetCStream() : |
713 * start a new compression job, using same parameters from previous job. |
748 * start a new compression job, using same parameters from previous job. |
714 * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. |
749 * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. |
715 * Note that zcs must be init at least once before using ZSTD_resetCStream(). |
750 * Note that zcs must be init at least once before using ZSTD_resetCStream(). |
716 * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. |
751 * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. |
717 * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. |
752 * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. |
718 * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, |
753 * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, |
719 * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. |
754 * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. |
720 * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ |
755 * @return : 0, or an error code (which can be tested using ZSTD_isError()) |
|
756 */ |
721 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); |
757 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); |
722 |
758 |
723 |
759 |
724 typedef struct { |
760 typedef struct { |
725 unsigned long long ingested; |
761 unsigned long long ingested; /* nb input bytes read and buffered */ |
726 unsigned long long consumed; |
762 unsigned long long consumed; /* nb input bytes actually compressed */ |
727 unsigned long long produced; |
763 unsigned long long produced; /* nb of compressed bytes generated and buffered */ |
|
764 unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ |
|
765 unsigned currentJobID; /* MT only : latest started job nb */ |
|
766 unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ |
728 } ZSTD_frameProgression; |
767 } ZSTD_frameProgression; |
729 |
768 |
730 /* ZSTD_getFrameProgression(): |
769 /* ZSTD_getFrameProgression() : |
731 * tells how much data has been ingested (read from input) |
770 * tells how much data has been ingested (read from input) |
732 * consumed (input actually compressed) and produced (output) for current frame. |
771 * consumed (input actually compressed) and produced (output) for current frame. |
733 * Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed. |
772 * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. |
734 * Can report progression inside worker threads (multi-threading and non-blocking mode). |
773 * Aggregates progression inside active worker threads. |
735 */ |
774 */ |
736 ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); |
775 ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); |
|
776 |
|
777 /*! ZSTD_toFlushNow() : |
|
778 * Tell how many bytes are ready to be flushed immediately. |
|
779 * Useful for multithreading scenarios (nbWorkers >= 1). |
|
780 * Probe the oldest active job, defined as oldest job not yet entirely flushed, |
|
781 * and check its output buffer. |
|
782 * @return : amount of data stored in oldest job and ready to be flushed immediately. |
|
783 * if @return == 0, it means either : |
|
784 * + there is no active job (could be checked with ZSTD_frameProgression()), or |
|
785 * + oldest job is still actively compressing data, |
|
786 * but everything it has produced has also been flushed so far, |
|
787 * therefore flushing speed is currently limited by production speed of oldest job |
|
788 * irrespective of the speed of concurrent newer jobs. |
|
789 */ |
|
790 ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); |
737 |
791 |
738 |
792 |
739 |
793 |
740 /*===== Advanced Streaming decompression functions =====*/ |
794 /*===== Advanced Streaming decompression functions =====*/ |
741 typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; |
795 typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; |
878 ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ |
932 ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ |
879 unsigned headerSize; |
933 unsigned headerSize; |
880 unsigned dictID; |
934 unsigned dictID; |
881 unsigned checksumFlag; |
935 unsigned checksumFlag; |
882 } ZSTD_frameHeader; |
936 } ZSTD_frameHeader; |
|
937 /** ZSTD_getFrameHeader() : |
|
938 * decode Frame Header, or requires larger `srcSize`. |
|
939 * @return : 0, `zfhPtr` is correctly filled, |
|
940 * >0, `srcSize` is too small, value is wanted `srcSize` amount, |
|
941 * or an error code, which can be tested using ZSTD_isError() */ |
883 ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ |
942 ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ |
884 ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ |
943 ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ |
885 |
944 |
886 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); |
945 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); |
887 ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); |
946 ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); |
899 |
958 |
900 /* ============================================ */ |
959 /* ============================================ */ |
901 /** New advanced API (experimental) */ |
960 /** New advanced API (experimental) */ |
902 /* ============================================ */ |
961 /* ============================================ */ |
903 |
962 |
904 /* notes on API design : |
963 /* API design : |
905 * In this proposal, parameters are pushed one by one into an existing context, |
964 * In this advanced API, parameters are pushed one by one into an existing context, |
906 * and then applied on all subsequent compression jobs. |
965 * using ZSTD_CCtx_set*() functions. |
907 * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT. |
966 * Pushed parameters are sticky : they are applied to next job, and any subsequent job. |
|
967 * It's possible to reset parameters to "default" using ZSTD_CCtx_reset(). |
|
968 * Important : "sticky" parameters only work with `ZSTD_compress_generic()` ! |
|
969 * For any other entry point, "sticky" parameters are ignored ! |
908 * |
970 * |
909 * This API is intended to replace all others advanced / experimental API entry points. |
971 * This API is intended to replace all others advanced / experimental API entry points. |
910 * But it stands a reasonable chance to become "stable", after a reasonable testing period. |
|
911 */ |
|
912 |
|
913 /* note on naming convention : |
|
914 * Initially, the API favored names like ZSTD_setCCtxParameter() . |
|
915 * In this proposal, convention is changed towards ZSTD_CCtx_setParameter() . |
|
916 * The main driver is that it identifies more clearly the target object type. |
|
917 * It feels clearer when considering multiple targets : |
|
918 * ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter()) |
|
919 * ZSTD_CCtxParams_setParameter() (rather than ZSTD_setCCtxParamsParameter() ) |
|
920 * etc... |
|
921 */ |
972 */ |
922 |
973 |
923 /* note on enum design : |
974 /* note on enum design : |
924 * All enum will be pinned to explicit values before reaching "stable API" status */ |
975 * All enum will be pinned to explicit values before reaching "stable API" status */ |
925 |
976 |
945 * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */ |
996 * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */ |
946 |
997 |
947 /* compression parameters */ |
998 /* compression parameters */ |
948 ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table |
999 ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table |
949 * Default level is ZSTD_CLEVEL_DEFAULT==3. |
1000 * Default level is ZSTD_CLEVEL_DEFAULT==3. |
950 * Special: value 0 means "do not change cLevel". |
1001 * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. |
951 * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type. |
1002 * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type. |
952 * Note 2 : setting a level sets all default values of other compression parameters. |
1003 * Note 2 : setting a level sets all default values of other compression parameters. |
953 * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ |
1004 * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ |
954 ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2. |
1005 ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2. |
955 * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. |
1006 * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. |
956 * Special: value 0 means "use default windowLog". |
1007 * Special: value 0 means "use default windowLog". |
957 * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27) |
1008 * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27) |
958 * requires explicitly allowing such window size during decompression stage. */ |
1009 * requires explicitly allowing such window size during decompression stage. */ |
959 ZSTD_p_hashLog, /* Size of the probe table, as a power of 2. |
1010 ZSTD_p_hashLog, /* Size of the initial probe table, as a power of 2. |
960 * Resulting table size is (1 << (hashLog+2)). |
1011 * Resulting table size is (1 << (hashLog+2)). |
961 * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. |
1012 * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. |
962 * Larger tables improve compression ratio of strategies <= dFast, |
1013 * Larger tables improve compression ratio of strategies <= dFast, |
963 * and improve speed of strategies > dFast. |
1014 * and improve speed of strategies > dFast. |
964 * Special: value 0 means "use default hashLog". */ |
1015 * Special: value 0 means "use default hashLog". */ |
965 ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2. |
1016 ZSTD_p_chainLog, /* Size of the multi-probe search table, as a power of 2. |
966 * Resulting table size is (1 << (chainLog+2)). |
1017 * Resulting table size is (1 << (chainLog+2)). |
|
1018 * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. |
967 * Larger tables result in better and slower compression. |
1019 * Larger tables result in better and slower compression. |
968 * This parameter is useless when using "fast" strategy. |
1020 * This parameter is useless when using "fast" strategy. |
|
1021 * Note it's still useful when using "dfast" strategy, |
|
1022 * in which case it defines a secondary probe table. |
969 * Special: value 0 means "use default chainLog". */ |
1023 * Special: value 0 means "use default chainLog". */ |
970 ZSTD_p_searchLog, /* Number of search attempts, as a power of 2. |
1024 ZSTD_p_searchLog, /* Number of search attempts, as a power of 2. |
971 * More attempts result in better and slower compression. |
1025 * More attempts result in better and slower compression. |
972 * This parameter is useless when using "fast" and "dFast" strategies. |
1026 * This parameter is useless when using "fast" and "dFast" strategies. |
973 * Special: value 0 means "use default searchLog". */ |
1027 * Special: value 0 means "use default searchLog". */ |
1045 |
1099 |
1046 /* =================================================================== */ |
1100 /* =================================================================== */ |
1047 /* experimental parameters - no stability guaranteed */ |
1101 /* experimental parameters - no stability guaranteed */ |
1048 /* =================================================================== */ |
1102 /* =================================================================== */ |
1049 |
1103 |
1050 ZSTD_p_compressLiterals=1000, /* control huffman compression of literals (enabled) by default. |
|
1051 * disabling it improves speed and decreases compression ratio by a large amount. |
|
1052 * note : this setting is automatically updated when changing compression level. |
|
1053 * positive compression levels set ZSTD_p_compressLiterals to 1. |
|
1054 * negative compression levels set ZSTD_p_compressLiterals to 0. */ |
|
1055 |
|
1056 ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, |
1104 ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, |
1057 * even when referencing into Dictionary content (default:0) */ |
1105 * even when referencing into Dictionary content (default:0) */ |
|
1106 ZSTD_p_forceAttachDict, /* ZSTD supports usage of a CDict in-place |
|
1107 * (avoiding having to copy the compression tables |
|
1108 * from the CDict into the working context). Using |
|
1109 * a CDict in this way saves an initial setup step, |
|
1110 * but comes at the cost of more work per byte of |
|
1111 * input. ZSTD has a simple internal heuristic that |
|
1112 * guesses which strategy will be faster. You can |
|
1113 * use this flag to override that guess. |
|
1114 * |
|
1115 * Note that the by-reference, in-place strategy is |
|
1116 * only used when reusing a compression context |
|
1117 * with compatible compression parameters. (If |
|
1118 * incompatible / uninitialized, the working |
|
1119 * context needs to be cleared anyways, which is |
|
1120 * about as expensive as overwriting it with the |
|
1121 * dictionary context, so there's no savings in |
|
1122 * using the CDict by-ref.) |
|
1123 * |
|
1124 * Values greater than 0 force attaching the dict. |
|
1125 * Values less than 0 force copying the dict. |
|
1126 * 0 selects the default heuristic-guided behavior. |
|
1127 */ |
1058 |
1128 |
1059 } ZSTD_cParameter; |
1129 } ZSTD_cParameter; |
1060 |
1130 |
1061 |
1131 |
1062 /*! ZSTD_CCtx_setParameter() : |
1132 /*! ZSTD_CCtx_setParameter() : |
1063 * Set one compression parameter, selected by enum ZSTD_cParameter. |
1133 * Set one compression parameter, selected by enum ZSTD_cParameter. |
1064 * Setting a parameter is generally only possible during frame initialization (before starting compression), |
1134 * Setting a parameter is generally only possible during frame initialization (before starting compression). |
1065 * except for a few exceptions which can be updated during compression: compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. |
1135 * Exception : when using multi-threading mode (nbThreads >= 1), |
1066 * Note : when `value` is an enum, cast it to unsigned for proper type checking. |
1136 * following parameters can be updated _during_ compression (within same frame): |
1067 * @result : informational value (typically, value being set clamped correctly), |
1137 * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. |
|
1138 * new parameters will be active on next job, or after a flush(). |
|
1139 * Note : when `value` type is not unsigned (int, or enum), cast it to unsigned for proper type checking. |
|
1140 * @result : informational value (typically, value being set, correctly clamped), |
1068 * or an error code (which can be tested with ZSTD_isError()). */ |
1141 * or an error code (which can be tested with ZSTD_isError()). */ |
1069 ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); |
1142 ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); |
|
1143 |
|
1144 /*! ZSTD_CCtx_getParameter() : |
|
1145 * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. |
|
1146 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
|
1147 */ |
|
1148 ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value); |
1070 |
1149 |
1071 /*! ZSTD_CCtx_setPledgedSrcSize() : |
1150 /*! ZSTD_CCtx_setPledgedSrcSize() : |
1072 * Total input data size to be compressed as a single frame. |
1151 * Total input data size to be compressed as a single frame. |
1073 * This value will be controlled at the end, and result in error if not respected. |
1152 * This value will be controlled at the end, and result in error if not respected. |
1074 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1153 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1112 * Note 2 : CDict is just referenced, its lifetime must outlive CCtx. */ |
1191 * Note 2 : CDict is just referenced, its lifetime must outlive CCtx. */ |
1113 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); |
1192 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); |
1114 |
1193 |
1115 /*! ZSTD_CCtx_refPrefix() : |
1194 /*! ZSTD_CCtx_refPrefix() : |
1116 * Reference a prefix (single-usage dictionary) for next compression job. |
1195 * Reference a prefix (single-usage dictionary) for next compression job. |
1117 * Decompression need same prefix to properly regenerate data. |
1196 * Decompression will need same prefix to properly regenerate data. |
1118 * Prefix is **only used once**. Tables are discarded at end of compression job. |
1197 * Compressing with a prefix is similar in outcome as performing a diff and compressing it, |
1119 * Subsequent compression jobs will be done without prefix (if none is explicitly referenced). |
1198 * but performs much faster, especially during decompression (compression speed is tunable with compression level). |
1120 * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead. |
1199 * Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end). |
1121 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1200 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1122 * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary |
1201 * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary |
1123 * Note 1 : Prefix buffer is referenced. It must outlive compression job. |
1202 * Note 1 : Prefix buffer is referenced. It **must** outlive compression job. |
1124 * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters. |
1203 * Its contain must remain unmodified up to end of compression (ZSTD_e_end). |
|
1204 * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, |
|
1205 * ensure that the window size is large enough to contain the entire source. |
|
1206 * See ZSTD_p_windowLog. |
|
1207 * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. |
1125 * It's a CPU consuming operation, with non-negligible impact on latency. |
1208 * It's a CPU consuming operation, with non-negligible impact on latency. |
1126 * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). |
1209 * If there is a need to use same prefix multiple times, consider loadDictionary instead. |
|
1210 * Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). |
1127 * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */ |
1211 * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */ |
1128 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); |
1212 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, |
1129 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); |
1213 const void* prefix, size_t prefixSize); |
|
1214 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, |
|
1215 const void* prefix, size_t prefixSize, |
|
1216 ZSTD_dictContentType_e dictContentType); |
1130 |
1217 |
1131 /*! ZSTD_CCtx_reset() : |
1218 /*! ZSTD_CCtx_reset() : |
1132 * Return a CCtx to clean state. |
1219 * Return a CCtx to clean state. |
1133 * Useful after an error, or to interrupt an ongoing compression job and start a new one. |
1220 * Useful after an error, or to interrupt an ongoing compression job and start a new one. |
1134 * Any internal data not yet flushed is cancelled. |
1221 * Any internal data not yet flushed is cancelled. |
|
1222 * The parameters and dictionary are kept unchanged, to reset them use ZSTD_CCtx_resetParameters(). |
|
1223 */ |
|
1224 ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); |
|
1225 |
|
1226 /*! ZSTD_CCtx_resetParameters() : |
|
1227 * All parameters are back to default values (compression level is ZSTD_CLEVEL_DEFAULT). |
1135 * Dictionary (if any) is dropped. |
1228 * Dictionary (if any) is dropped. |
1136 * All parameters are back to default values. |
1229 * Resetting parameters is only possible during frame initialization (before starting compression). |
1137 * It's possible to modify compression parameters after a reset. |
1230 * To reset the context use ZSTD_CCtx_reset(). |
1138 */ |
1231 * @return 0 or an error code (which can be checked with ZSTD_isError()). |
1139 ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); |
1232 */ |
|
1233 ZSTDLIB_API size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx); |
1140 |
1234 |
1141 |
1235 |
1142 |
1236 |
1143 typedef enum { |
1237 typedef enum { |
1144 ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */ |
1238 ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */ |
1233 * Note : when `value` is an enum, cast it to unsigned for proper type checking. |
1327 * Note : when `value` is an enum, cast it to unsigned for proper type checking. |
1234 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1328 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1235 */ |
1329 */ |
1236 ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value); |
1330 ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value); |
1237 |
1331 |
|
1332 /*! ZSTD_CCtxParam_getParameter() : |
|
1333 * Similar to ZSTD_CCtx_getParameter. |
|
1334 * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. |
|
1335 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
|
1336 */ |
|
1337 ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned* value); |
|
1338 |
1238 /*! ZSTD_CCtx_setParametersUsingCCtxParams() : |
1339 /*! ZSTD_CCtx_setParametersUsingCCtxParams() : |
1239 * Apply a set of ZSTD_CCtx_params to the compression context. |
1340 * Apply a set of ZSTD_CCtx_params to the compression context. |
1240 * This can be done even after compression is started, |
1341 * This can be done even after compression is started, |
1241 * if nbWorkers==0, this will have no impact until a new compression is started. |
1342 * if nbWorkers==0, this will have no impact until a new compression is started. |
1242 * if nbWorkers>=1, new parameters will be picked up at next job, |
1343 * if nbWorkers>=1, new parameters will be picked up at next job, |
1244 */ |
1345 */ |
1245 ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( |
1346 ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( |
1246 ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); |
1347 ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); |
1247 |
1348 |
1248 |
1349 |
1249 /*=== Advanced parameters for decompression API ===*/ |
1350 /* ==================================== */ |
1250 |
1351 /*=== Advanced decompression API ===*/ |
1251 /* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object, |
1352 /* ==================================== */ |
1252 * but before starting decompression of a frame. |
1353 |
|
1354 /* The following API works the same way as the advanced compression API : |
|
1355 * a context is created, parameters are pushed into it one by one, |
|
1356 * then the context can be used to decompress data using an interface similar to the straming API. |
1253 */ |
1357 */ |
1254 |
1358 |
1255 /*! ZSTD_DCtx_loadDictionary() : |
1359 /*! ZSTD_DCtx_loadDictionary() : |
1256 * Create an internal DDict from dict buffer, |
1360 * Create an internal DDict from dict buffer, |
1257 * to be used to decompress next frames. |
1361 * to be used to decompress next frames. |
1284 ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); |
1388 ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); |
1285 |
1389 |
1286 |
1390 |
1287 /*! ZSTD_DCtx_refPrefix() : |
1391 /*! ZSTD_DCtx_refPrefix() : |
1288 * Reference a prefix (single-usage dictionary) for next compression job. |
1392 * Reference a prefix (single-usage dictionary) for next compression job. |
1289 * Prefix is **only used once**. It must be explicitly referenced before each frame. |
1393 * This is the reverse operation of ZSTD_CCtx_refPrefix(), |
1290 * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead. |
1394 * and must use the same prefix as the one used during compression. |
|
1395 * Prefix is **only used once**. Reference is discarded at end of frame. |
|
1396 * End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0. |
1291 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1397 * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1292 * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary |
1398 * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary |
1293 * Note 2 : Prefix buffer is referenced. It must outlive compression job. |
1399 * Note 2 : Prefix buffer is referenced. It **must** outlive decompression job. |
|
1400 * Prefix buffer must remain unmodified up to the end of frame, |
|
1401 * reached when ZSTD_DCtx_decompress_generic() returns 0. |
1294 * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). |
1402 * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). |
1295 * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. |
1403 * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. |
1296 * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. |
1404 * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. |
1297 */ |
1405 * A fulldict prefix is more costly though. |
1298 ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); |
1406 */ |
1299 ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); |
1407 ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, |
|
1408 const void* prefix, size_t prefixSize); |
|
1409 ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, |
|
1410 const void* prefix, size_t prefixSize, |
|
1411 ZSTD_dictContentType_e dictContentType); |
1300 |
1412 |
1301 |
1413 |
1302 /*! ZSTD_DCtx_setMaxWindowSize() : |
1414 /*! ZSTD_DCtx_setMaxWindowSize() : |
1303 * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. |
1415 * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. |
1304 * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario). |
1416 * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario). |
1314 * This instruction is mandatory to decode data without a fully-formed header, |
1426 * This instruction is mandatory to decode data without a fully-formed header, |
1315 * such ZSTD_f_zstd1_magicless for example. |
1427 * such ZSTD_f_zstd1_magicless for example. |
1316 * @return : 0, or an error code (which can be tested using ZSTD_isError()). |
1428 * @return : 0, or an error code (which can be tested using ZSTD_isError()). |
1317 */ |
1429 */ |
1318 ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); |
1430 ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); |
|
1431 |
|
1432 |
|
1433 /*! ZSTD_getFrameHeader_advanced() : |
|
1434 * same as ZSTD_getFrameHeader(), |
|
1435 * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ |
|
1436 ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, |
|
1437 const void* src, size_t srcSize, ZSTD_format_e format); |
1319 |
1438 |
1320 |
1439 |
1321 /*! ZSTD_decompress_generic() : |
1440 /*! ZSTD_decompress_generic() : |
1322 * Behave the same as ZSTD_decompressStream. |
1441 * Behave the same as ZSTD_decompressStream. |
1323 * Decompression parameters cannot be changed once decompression is started. |
1442 * Decompression parameters cannot be changed once decompression is started. |
1381 + In case of multiple successive blocks, should some of them be uncompressed, |
1500 + In case of multiple successive blocks, should some of them be uncompressed, |
1382 decoder must be informed of their existence in order to follow proper history. |
1501 decoder must be informed of their existence in order to follow proper history. |
1383 Use ZSTD_insertBlock() for such a case. |
1502 Use ZSTD_insertBlock() for such a case. |
1384 */ |
1503 */ |
1385 |
1504 |
1386 #define ZSTD_BLOCKSIZELOG_MAX 17 |
|
1387 #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */ |
|
1388 /*===== Raw zstd block functions =====*/ |
1505 /*===== Raw zstd block functions =====*/ |
1389 ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); |
1506 ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); |
1390 ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
1507 ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
1391 ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
1508 ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
1392 ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ |
1509 ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ |