diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 63704e67e..bb6f0eac9 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -332,7 +332,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
typedef struct {
unsigned long long frameContentSize;
- unsigned windowSize;
+ size_t windowSize;
unsigned dictID;
unsigned checksumFlag;
} ZSTD_frameHeader;
@@ -389,6 +389,12 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
however it does mean that all frame data must be present and valid.
+size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+ `src` should point to the start of a ZSTD frame
+ `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
+ @return : size of the Frame Header
+
+
Context memory usage
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@@ -401,25 +407,41 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
Object memory usage can evolve if it's re-used multiple times.
-size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateCCtxSize(int compressionLevel);
+size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDCtxSize(void);
These functions make it possible to estimate memory usage
- of a future target object, before its allocation,
- given a set of parameters, which vary depending on target object.
- The objective is to guide decision before allocation.
+ of a future {D,C}Ctx, before its creation.
+ ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one.
+ It will also consider src size to be arbitrarily "large", which is worst case.
+ If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
+ ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
Note : CCtx estimation is only correct for single-threaded compression
-size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
-size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
- Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
- an internal ?Dict will be created, which size is not estimated here.
+
size_t ZSTD_estimateCStreamSize(int compressionLevel);
+size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateDStreamSize(size_t windowSize);
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+ ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one.
+ It will also consider src size to be arbitrarily "large", which is worst case.
+ If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
+ ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ Note : CStream estimation is only correct for single-threaded compression.
+ ZSTD_DStream memory budget depends on window Size.
+ This information can be passed manually, using ZSTD_estimateDStreamSize,
+ or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
+ Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ an internal ?Dict will be created, which additional size is not estimated here.
In this case, get total size by adding ZSTD_estimate?DictSize
-size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
- Note : dictionary created "byReference" are smaller
+
ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
+ Note : dictionary created "byReference" are smaller
Advanced compression functions
@@ -606,6 +628,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
Advanced streaming functions
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);
/**< same as ZSTD_initStaticCCtx() */
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
@@ -624,6 +647,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
Advanced Streaming decompression functions
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
@@ -678,16 +702,18 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
- It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
- such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
- and the dictionary ID used.
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+ such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
+ and the dictionary ID in use.
(Note : content size is optional, it may not be present. 0 means : content size unknown).
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
- Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
- Frame parameters are extracted from the beginning of the compressed frame.
- Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
- @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+ Each application can set its own limit, depending on local restrictions.
+ For extended interoperability, it is recommended to support windowSize of at least 8 MB.
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+ Data fragment must be large enough to ensure successful decoding.
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+ @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
@@ -731,7 +757,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
It also returns Frame Size as fparamsPtr->frameContentSize.
-Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);
/**< doesn't consume input, see details below */
+Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);
/**< doesn't consume input */
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e53496091..5c2035a24 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -532,7 +532,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
}
-size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
+size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams)
{
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.searchLength==3) ? 3 : 4;
@@ -558,9 +558,15 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
return sizeof(ZSTD_CCtx) + neededSpace;
}
-size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
+size_t ZSTD_estimateCCtxSize(int compressionLevel)
{
- size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
+ return ZSTD_estimateCCtxSize_advanced(cParams);
+}
+
+size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams)
+{
+ size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced(cParams);
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
@@ -569,6 +575,11 @@ size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
return CCtxSize + streamingSize;
}
+size_t ZSTD_estimateCStreamSize(int compressionLevel) {
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
+ return ZSTD_estimateCStreamSize_advanced(cParams);
+}
+
static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
ZSTD_compressionParameters cParams2)
@@ -3350,16 +3361,22 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/* ===== Dictionary API ===== */
-/*! ZSTD_estimateCDictSize() :
+/*! ZSTD_estimateCDictSize_advanced() :
* Estimate amount of memory that will be needed to create a dictionary with following arguments */
-size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
+size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference)
{
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
- DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize(cParams));
- return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
+ DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams));
+ return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams)
+ (byReference ? 0 : dictSize);
}
+size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
+{
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
+ return ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
+}
+
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support sizeof on NULL */
@@ -3482,7 +3499,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams)
{
- size_t const cctxSize = ZSTD_estimateCCtxSize(cParams);
+ size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams);
size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize)
+ cctxSize;
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
@@ -3577,6 +3594,11 @@ ZSTD_CStream* ZSTD_createCStream(void)
return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
}
+ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize)
+{
+ return ZSTD_initStaticCCtx(workspace, workspaceSize);
+}
+
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
{ /* CStream and CCtx are now same object */
return ZSTD_createCCtx_advanced(customMem);
@@ -3970,30 +3992,30 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
-{ /* "default" */
+{ /* "default" - guarantees a monotonically increasing memory budget */
/* W, C, H, S, L, TL, strat */
{ 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */
{ 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */
- { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/
- { 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/
- { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */
- { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
- { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */
+ { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */
+ { 20, 17, 18, 1, 5, 16, ZSTD_dfast }, /* level 4 */
+ { 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */
+ { 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
+ { 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */
{ 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */
- { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */
+ { 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */
{ 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */
{ 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */
{ 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */
{ 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */
- { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */
+ { 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */
{ 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */
- { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
+ { 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
{ 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */
{ 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */
- { 26, 26, 23, 7, 3,256, ZSTD_btultra }, /* level 21 */
+ { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */
{ 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */
},
{ /* for srcSize <= 256 KB */
@@ -4076,6 +4098,43 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
},
};
+/* This function just controls
+ * the monotonic memory budget increase of ZSTD_defaultCParameters[0].
+ * Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
+ */
+MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
+{
+# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
+# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
+# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
+ assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \
+ assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
+ }
+
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
+ ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
+ assert(ZSTD_maxCLevel()==22);
+}
+
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
* Size values are optional, provide 0 if not known or unused */
@@ -4084,6 +4143,15 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
size_t const addedSize = srcSizeHint ? 0 : 500;
U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
+
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+ static int g_monotonicTest = 1;
+ if (g_monotonicTest) {
+ ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget();
+ g_monotonicTest=0;
+ }
+#endif
+
if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
{ ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 95d18d4b5..e823c3dbf 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1282,7 +1282,7 @@ static size_t ZSTD_decompressSequencesLong(
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
- unsigned const windowSize = dctx->fParams.windowSize;
+ unsigned const windowSize32 = (unsigned)dctx->fParams.windowSize;
int nbSeq;
/* Build Decoding Tables */
@@ -1312,13 +1312,13 @@ static size_t ZSTD_decompressSequencesLong(
/* prepare in advance */
for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb0) return ERROR(srcSize_wrong);
+ return ZSTD_estimateDStreamSize(fh.windowSize);
+}
+
/* ***** Decompression ***** */
diff --git a/lib/zstd.h b/lib/zstd.h
index 8cf9ba75f..75bdb4895 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -412,7 +412,7 @@ typedef struct {
typedef struct {
unsigned long long frameContentSize;
- unsigned windowSize;
+ size_t windowSize;
unsigned dictID;
unsigned checksumFlag;
} ZSTD_frameHeader;
@@ -495,23 +495,39 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/*! ZSTD_estimate*() :
* These functions make it possible to estimate memory usage
- * of a future target object, before its allocation,
- * given a set of parameters, which vary depending on target object.
- * The objective is to guide decision before allocation.
+ * of a future {D,C}Ctx, before its creation.
+ * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
+ * It will also consider src size to be arbitrarily "large", which is worst case.
+ * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
+ * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* Note : CCtx estimation is only correct for single-threaded compression */
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
/*! ZSTD_estimate?StreamSize() :
+ * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
+ * It will also consider src size to be arbitrarily "large", which is worst case.
+ * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
+ * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ * Note : CStream estimation is only correct for single-threaded compression.
+ * ZSTD_DStream memory budget depends on window Size.
+ * This information can be passed manually, using ZSTD_estimateDStreamSize,
+ * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
- * an internal ?Dict will be created, which size is not estimated here.
+ * an internal ?Dict will be created, which additional size is not estimated here.
* In this case, get total size by adding ZSTD_estimate?DictSize */
-ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
-ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
/*! ZSTD_estimate?DictSize() :
+ * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ * ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
* Note : dictionary created "byReference" are smaller */
-ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
@@ -707,6 +723,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
@@ -727,6 +744,7 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
/*===== Advanced Streaming decompression functions =====*/
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
@@ -793,16 +811,18 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
- It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
- such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
- and the dictionary ID used.
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+ such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
+ and the dictionary ID in use.
(Note : content size is optional, it may not be present. 0 means : content size unknown).
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
- Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
- Frame parameters are extracted from the beginning of the compressed frame.
- Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
- @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+ Each application can set its own limit, depending on local restrictions.
+ For extended interoperability, it is recommended to support windowSize of at least 8 MB.
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+ Data fragment must be large enough to ensure successful decoding.
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+ @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
@@ -847,7 +867,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
*/
/*===== Buffer-less streaming decompression functions =====*/
-ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
+ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a3a56d9d8..0dcba3cea 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -193,8 +193,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Static CCtx tests */
#define STATIC_CCTX_LEVEL 3
DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
- { ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
- size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
+ { size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
void* const staticCCtxBuffer = malloc(staticCCtxSize);
size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
void* const staticDCtxBuffer = malloc(staticDCtxSize);
@@ -502,7 +501,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
- size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1 /*byReference*/);
+ size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 1 /*byReference*/);
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
}
@@ -535,7 +534,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
- size_t const cdictSize = ZSTD_estimateCDictSize(cParams, dictSize, 0);
+ size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
void* const cdictBuffer = malloc(cdictSize);
if (cdictBuffer==NULL) goto _output_error;
{ ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 1185c6649..da06ccb52 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -390,8 +390,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
- size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params);
- size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params);
+ size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_advanced(params);
+ size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_advanced(winners[cLevel].params);
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0e14fd23a..aae83972a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -210,9 +210,9 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* context size functions */
DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
- size_t const s = ZSTD_estimateCStreamSize(cParams)
+ size_t const s = ZSTD_estimateCStreamSize_advanced(cParams)
/* uses ZSTD_initCStream_usingDict() */
- + ZSTD_estimateCDictSize(cParams, dictSize, 0);
+ + ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
}
@@ -285,8 +285,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
if (gfhError!=0) goto _output_error;
- DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
- { size_t const s = ZSTD_estimateDStreamSize(fhi)
+ DISPLAYLEVEL(5, " (windowSize : %u) ", (U32)fhi.windowSize);
+ { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
/* uses ZSTD_initDStream_usingDict() */
+ ZSTD_estimateDDictSize(dictSize, 0);
if (ZSTD_isError(s)) goto _output_error;