Deprecated bufferless and block level APIs
* Mark all bufferless and block level functions as deprecated * Update documentation to suggest not using these functions * Add `_deprecated()` wrappers for functions that we use internally and call those instead
This commit is contained in:
Nick Terrell
committed by
Nick Terrell
Nick Terrell
parent
53bad103ce
commit
fbd97f305a
+238
-209
@@ -1389,7 +1389,7 @@ typedef enum {
|
||||
} ZSTD_paramSwitch_e;
|
||||
|
||||
/***************************************
|
||||
* Frame size functions
|
||||
* Frame header and size functions
|
||||
***************************************/
|
||||
|
||||
/*! ZSTD_findDecompressedSize() :
|
||||
@@ -1436,6 +1436,30 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
|
||||
* or an error code (if srcSize is too small) */
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
||||
|
||||
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
|
||||
typedef struct {
|
||||
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
|
||||
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
|
||||
unsigned blockSizeMax;
|
||||
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
|
||||
unsigned headerSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
unsigned _reserved1;
|
||||
unsigned _reserved2;
|
||||
} ZSTD_frameHeader;
|
||||
|
||||
/*! ZSTD_getFrameHeader() :
|
||||
* decode Frame Header, or requires larger `srcSize`.
|
||||
* @return : 0, `zfhPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
/*! ZSTD_getFrameHeader_advanced() :
|
||||
* same as ZSTD_getFrameHeader(),
|
||||
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
|
||||
|
||||
/*! ZSTD_decompressionMargin() :
|
||||
* Zstd supports in-place decompression, where the input and output buffers overlap.
|
||||
* In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
|
||||
@@ -2608,214 +2632,6 @@ ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
||||
|
||||
|
||||
/*********************************************************************
|
||||
* Buffer-less and synchronous inner streaming functions
|
||||
*
|
||||
* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
||||
* But it's also a complex one, with several restrictions, documented below.
|
||||
* Prefer normal streaming API for an easier experience.
|
||||
********************************************************************* */
|
||||
|
||||
/**
|
||||
Buffer-less streaming compression (synchronous mode)
|
||||
|
||||
A ZSTD_CCtx object is required to track streaming operations.
|
||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||
|
||||
Start by initializing a context.
|
||||
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
|
||||
|
||||
Then, consume your input using ZSTD_compressContinue().
|
||||
There are some important considerations to keep in mind when using this advanced function :
|
||||
- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
|
||||
- Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
|
||||
- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
|
||||
Worst case evaluation is provided by ZSTD_compressBound().
|
||||
ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
|
||||
- ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
|
||||
It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
|
||||
- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
|
||||
In which case, it will "discard" the relevant memory section from its history.
|
||||
|
||||
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
|
||||
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
|
||||
Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
|
||||
|
||||
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
|
||||
*/
|
||||
|
||||
/*===== Buffer-less streaming compression functions =====*/
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
|
||||
|
||||
ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
|
||||
ZSTDLIB_STATIC_API
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
|
||||
ZSTD_DEPRECATED("use advanced API to access custom parameters")
|
||||
ZSTDLIB_STATIC_API
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
ZSTD_DEPRECATED("use advanced API to access custom parameters")
|
||||
ZSTDLIB_STATIC_API
|
||||
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
/**
|
||||
Buffer-less streaming decompression (synchronous mode)
|
||||
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
||||
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
||||
Data fragment must be large enough to ensure successful decoding.
|
||||
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
||||
result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTD_isError().
|
||||
|
||||
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
||||
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
|
||||
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
|
||||
As a consequence, check that values remain within valid application range.
|
||||
For example, do not allocate memory blindly, check that `windowSize` is within expectation.
|
||||
Each application can set its own limits, depending on local restrictions.
|
||||
For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
|
||||
|
||||
ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
|
||||
ZSTD_decompressContinue() is very sensitive to contiguity,
|
||||
if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
|
||||
or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
|
||||
There are multiple ways to guarantee this condition.
|
||||
|
||||
The most memory efficient way is to use a round buffer of sufficient size.
|
||||
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
|
||||
which can return an error code if required value is too large for current system (in 32-bits mode).
|
||||
In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
|
||||
up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
|
||||
which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
|
||||
At which point, decoding can resume from the beginning of the buffer.
|
||||
Note that already decoded data stored in the buffer should be flushed before being overwritten.
|
||||
|
||||
There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
|
||||
|
||||
Finally, if you control the compression process, you can also ignore all buffer size rules,
|
||||
as long as the encoder and decoder progress in "lock-step",
|
||||
aka use exactly the same buffer sizes, break contiguity at the same place, etc.
|
||||
|
||||
Once buffers are setup, start decompression, with ZSTD_decompressBegin().
|
||||
If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
|
||||
|
||||
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
|
||||
|
||||
result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
||||
It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
|
||||
It can also be an error code, which can be tested with ZSTD_isError().
|
||||
|
||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
|
||||
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
||||
This information is not required to properly decode a frame.
|
||||
|
||||
== Special case : skippable frames ==
|
||||
|
||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by decompressor.
|
||||
The format of skippable frames is as follows :
|
||||
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
|
||||
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
|
||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||
For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
|
||||
For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
|
||||
*/
|
||||
|
||||
/*===== Buffer-less streaming decompression functions =====*/
|
||||
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
|
||||
typedef struct {
|
||||
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
|
||||
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
|
||||
unsigned blockSizeMax;
|
||||
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
|
||||
unsigned headerSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
unsigned _reserved1;
|
||||
unsigned _reserved2;
|
||||
} ZSTD_frameHeader;
|
||||
|
||||
/*! ZSTD_getFrameHeader() :
|
||||
* decode Frame Header, or requires larger `srcSize`.
|
||||
* @return : 0, `zfhPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
/*! ZSTD_getFrameHeader_advanced() :
|
||||
* same as ZSTD_getFrameHeader(),
|
||||
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* misc */
|
||||
ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
|
||||
ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||
ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
|
||||
|
||||
|
||||
|
||||
/* ============================ */
|
||||
/** Block level API */
|
||||
/* ============================ */
|
||||
|
||||
/*!
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Compressing and decompressing require a context structure
|
||||
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
|
||||
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
|
||||
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
||||
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
||||
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
|
||||
===> In which case, nothing is produced into `dst` !
|
||||
+ User __must__ test for such outcome and deal directly with uncompressed data
|
||||
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
|
||||
Doing so would mess up with statistics history, leading to potential data corruption.
|
||||
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
|
||||
+ In case of multiple successive blocks, should some of them be uncompressed,
|
||||
decoder must be informed of their existence in order to follow proper history.
|
||||
Use ZSTD_insertBlock() for such a case.
|
||||
*/
|
||||
|
||||
/*===== Raw zstd block functions =====*/
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
|
||||
|
||||
|
||||
/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
|
||||
*
|
||||
* *** OVERVIEW ***
|
||||
@@ -2977,6 +2793,219 @@ ZSTD_registerSequenceProducer(
|
||||
ZSTD_sequenceProducer_F* sequenceProducer
|
||||
);
|
||||
|
||||
|
||||
/*********************************************************************
|
||||
* Buffer-less and synchronous inner streaming functions (DEPRECATED)
|
||||
*
|
||||
* This API is deprecated, and will be removed in a future version.
|
||||
* It allows streaming (de)compression with user allocated buffers.
|
||||
* However, it is hard to use, and not as well tested as the rest of
|
||||
* our API.
|
||||
*
|
||||
* Please use the normal streaming API instead: ZSTD_compressStream2,
|
||||
* and ZSTD_decompressStream.
|
||||
* If there is functionality that you need, but it doesn't provide,
|
||||
* please open an issue on our GitHub.
|
||||
********************************************************************* */
|
||||
|
||||
/**
|
||||
Buffer-less streaming compression (synchronous mode)
|
||||
|
||||
A ZSTD_CCtx object is required to track streaming operations.
|
||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||
|
||||
Start by initializing a context.
|
||||
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
|
||||
|
||||
Then, consume your input using ZSTD_compressContinue().
|
||||
There are some important considerations to keep in mind when using this advanced function :
|
||||
- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
|
||||
- Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
|
||||
- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
|
||||
Worst case evaluation is provided by ZSTD_compressBound().
|
||||
ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
|
||||
- ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
|
||||
It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
|
||||
- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
|
||||
In which case, it will "discard" the relevant memory section from its history.
|
||||
|
||||
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
|
||||
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
|
||||
Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
|
||||
|
||||
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
|
||||
*/
|
||||
|
||||
/*===== Buffer-less streaming compression functions =====*/
|
||||
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
|
||||
|
||||
ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
|
||||
ZSTDLIB_STATIC_API
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
|
||||
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
|
||||
ZSTD_DEPRECATED("use advanced API to access custom parameters")
|
||||
ZSTDLIB_STATIC_API
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
ZSTD_DEPRECATED("use advanced API to access custom parameters")
|
||||
ZSTDLIB_STATIC_API
|
||||
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
/**
|
||||
Buffer-less streaming decompression (synchronous mode)
|
||||
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
||||
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
||||
Data fragment must be large enough to ensure successful decoding.
|
||||
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
||||
result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTD_isError().
|
||||
|
||||
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
||||
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
|
||||
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
|
||||
As a consequence, check that values remain within valid application range.
|
||||
For example, do not allocate memory blindly, check that `windowSize` is within expectation.
|
||||
Each application can set its own limits, depending on local restrictions.
|
||||
For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
|
||||
|
||||
ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
|
||||
ZSTD_decompressContinue() is very sensitive to contiguity,
|
||||
if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
|
||||
or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
|
||||
There are multiple ways to guarantee this condition.
|
||||
|
||||
The most memory efficient way is to use a round buffer of sufficient size.
|
||||
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
|
||||
which can return an error code if required value is too large for current system (in 32-bits mode).
|
||||
In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
|
||||
up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
|
||||
which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
|
||||
At which point, decoding can resume from the beginning of the buffer.
|
||||
Note that already decoded data stored in the buffer should be flushed before being overwritten.
|
||||
|
||||
There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
|
||||
|
||||
Finally, if you control the compression process, you can also ignore all buffer size rules,
|
||||
as long as the encoder and decoder progress in "lock-step",
|
||||
aka use exactly the same buffer sizes, break contiguity at the same place, etc.
|
||||
|
||||
Once buffers are setup, start decompression, with ZSTD_decompressBegin().
|
||||
If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
|
||||
|
||||
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
|
||||
|
||||
result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
||||
It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
|
||||
It can also be an error code, which can be tested with ZSTD_isError().
|
||||
|
||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
|
||||
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
|
||||
This information is not required to properly decode a frame.
|
||||
|
||||
== Special case : skippable frames ==
|
||||
|
||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by decompressor.
|
||||
The format of skippable frames is as follows :
|
||||
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
|
||||
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
|
||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||
For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
|
||||
For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
|
||||
*/
|
||||
|
||||
/*===== Buffer-less streaming decompression functions =====*/
|
||||
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/* misc */
|
||||
ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
|
||||
ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||
ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
||||
|
||||
|
||||
|
||||
|
||||
/* ========================================= */
|
||||
/** Block level API (DEPRECATED) */
|
||||
/* ========================================= */
|
||||
|
||||
/*!
|
||||
|
||||
This API is deprecated in favor of the regular compression API.
|
||||
You can get the frame header down to 2 bytes by setting:
|
||||
- ZSTD_c_format = ZSTD_f_zstd1_magicless
|
||||
- ZSTD_c_contentSizeFlag = 0
|
||||
- ZSTD_c_checksumFlag = 0
|
||||
- ZSTD_c_dictIDFlag = 0
|
||||
|
||||
This API is not as well tested as our normal API, so we recommend not using it.
|
||||
We will be removing it in a future version. If the normal API doesn't provide
|
||||
the functionality you need, please open a GitHub issue.
|
||||
|
||||
Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Compressing and decompressing require a context structure
|
||||
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
|
||||
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
|
||||
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
||||
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
||||
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
||||
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
|
||||
===> In which case, nothing is produced into `dst` !
|
||||
+ User __must__ test for such outcome and deal directly with uncompressed data
|
||||
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
|
||||
Doing so would mess up with statistics history, leading to potential data corruption.
|
||||
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
|
||||
+ In case of multiple successive blocks, should some of them be uncompressed,
|
||||
decoder must be informed of their existence in order to follow proper history.
|
||||
Use ZSTD_insertBlock() for such a case.
|
||||
*/
|
||||
|
||||
/*===== Raw zstd block functions =====*/
|
||||
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
|
||||
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
|
||||
|
||||
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
Reference in New Issue
Block a user