ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added : zstd buffered API

Browse Source
This commit is contained in:
Yann Collet
2015-11-25 14:42:45 +01:00
parent c36521571e
commit 88fcd2916e
14 changed files with 1177 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_static.h
+132 -109
View File
@@ -52,71 +52,9 @@ extern "C" {
/* *************************************
* Types
***************************************/
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
typedef struct
{
U32 windowLog; /* largest match distance : impact decompression buffer size */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 searchLog; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */
ZSTD_strategy strategy;
} ZSTD_parameters;
/* *************************************
* Advanced function
***************************************/
/** ZSTD_compress_advanced
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
ZSTD_parameters params);
/** ZSTD_validateParams
correct params value to remain within authorized range
srcSizeHint value is optional, select 0 if not known */
void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint);
/* *************************************
* Streaming functions
***************************************/
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/*
Use above functions alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
Result is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
*/
/* *************************************
* Prefix - version detection
***************************************/
#define ZSTD_magicNumber 0xFD2FB523 /* v0.3 (current)*/
/* *************************************
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
#define ZSTD_WINDOWLOG_MAX 26
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_HASHLOG_MAX 28
@@ -126,55 +64,140 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
#define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 4
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
typedef struct
{
U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster */
U32 searchLog; /* nb of searches : larger == more compression, slower */
U32 searchLength; /* size of matches : larger == faster decompression, sometimes less compression */
ZSTD_strategy strategy;
} ZSTD_parameters;
/* *************************************
* Advanced function
***************************************/
/** ZSTD_getParams
* return ZSTD_parameters structure for a selected compression level and srcSize.
* srcSizeHint value is optional, select 0 if not known */
ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
/** ZSTD_validateParams
* correct params value to remain within authorized range */
void ZSTD_validateParams(ZSTD_parameters* params);
/** ZSTD_compress_advanced
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
ZSTD_parameters params);
/* **************************************
* Streaming functions (bufferless mode)
****************************************/
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/**
Streaming decompression, bufferless mode
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status.
First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
This function doesn't consume its input. It needs enough input data to properly decode the frame header.
The objective is to retrieve *params.windowlog, to know how much memory is required during decoding.
Result : 0 if successfull, it means the ZSTD_parameters structure has been filled.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
Then it's possible to start decompression.
Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() will use previous data blocks during decompress.
They should be located contiguously prior to current block. Alternatively, a round buffer is possible.
Just make sure that the combined of current and accessible past blocks is a minimum of (1 << windowlog).
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
*/
/* *************************************
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
{ /* for <= 128 KB */
/* W, C, H, S, L, strat */
{ 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
{ 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
{ 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
{ 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
{ 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
{ 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
{ 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
{ 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
{ 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
{ 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
{ 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
{ 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
{ 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
{ 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
{ 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
{ 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
{ 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
{ 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
{ 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
{ 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
{ /* "default" */
/* W, C, H, S, L, strat */
{ 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
{ 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
{ 0, 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
{ 0, 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
{ 0, 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */
{ 0, 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */
{ 0, 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
{ 0, 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
{ 0, 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
{ 0, 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */
{ 0, 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
{ 0, 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
{ 0, 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
{ 0, 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
{ 0, 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
{ 0, 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
{ 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
{ 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
{ 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
{ 0, 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */
},
{ /* for > 128 KB */
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, strat */
{ 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
{ 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
{ 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
{ 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */
{ 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */
{ 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
{ 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
{ 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
{ 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */
{ 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
{ 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
{ 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
{ 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
{ 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
{ 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
{ 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
{ 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
{ 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */
}
{ 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
{ 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
{ 0, 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
{ 0, 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
{ 0, 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
{ 0, 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
{ 0, 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
{ 0, 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
{ 0, 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
{ 0, 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
{ 0, 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
{ 0, 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
{ 0, 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
{ 0, 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
{ 0, 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
{ 0, 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
{ 0, 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
{ 0, 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
{ 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
{ 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
},
};