size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); -Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. - `frameContentSize` is an optional parameter, provide `0` if unknown -
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);Create a ZSTD compression context using external alloc and free functions
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); +Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. + Set streaming to 1 if the CCtx will be used for streaming (CStream). + Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict. + Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size +
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);amount of used memory is variable, depending primarily on compression level
Create a ZSTD_CDict using external alloc and free, and customized compression parameters
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize); +Estimate amount of memory that will be needed to create a dictionary with following arguments + Note : if dictionary is created "byReference", reduce this amount by dictSize +
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);Gives the amount of memory used by a given ZSTD_sizeof_CDict
Create a ZSTD_DDict using external alloc and free, optionally by reference
size_t ZSTD_estimateDDictSize(size_t dictSize); +Estimate amount of memory that will be needed to create a dictionary for decompression. + Note : if dictionary is created "byReference", reduce this amount by dictSize +
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);Gives the amount of memory used by a given ZSTD_DDict
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);/**< same as ZSTD_sizeof_CCtx */ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ -size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */ +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 748aff930..c5687b21c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2984,6 +2984,14 @@ struct ZSTD_CDict_s { ZSTD_CCtx* refContext; }; /* typedef'd tp ZSTD_CDict within "zstd.h" */ +/*! ZSTD_estimateCDictSize() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize) +{ + cParams = ZSTD_adjustCParams(cParams, 0, dictSize); + return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0); +} + size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 910f9ab78..88488b4b0 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1969,6 +1969,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) } } +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : if dictionary is created "byReference", reduce this amount by dictSize */ +size_t ZSTD_estimateDDictSize(size_t dictSize) +{ + return dictSize + sizeof(ZSTD_DDict); +} + size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; /* support sizeof on NULL */ diff --git a/lib/zstd.h b/lib/zstd.h index e8614334b..270943b30 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -458,17 +458,17 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t /*************************************** * Advanced compression functions ***************************************/ -/*! ZSTD_estimateCCtxSize() : - * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. - * Set streaming to 1 if the CCtx will be used for streaming (CStream) - * Note : this function is currently unable to estimate additional memory allocation needed to create an internal CDict - * which can only happen when starting with ZSTD_initCStream_usingDict() */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); - /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_estimateCCtxSize() : + * Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. + * Set streaming to 1 if the CCtx will be used for streaming (CStream). + * Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict. + * Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); + /*! ZSTD_sizeofCCtx() : * amount of used memory is variable, depending primarily on compression level */ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); @@ -493,6 +493,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_compressionParameters cParams, ZSTD_customMem customMem); +/*! ZSTD_estimateCDictSize() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments + * Note : if dictionary is created "byReference", reduce this amount by dictSize */ +ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize); + /*! ZSTD_sizeof_CDict() : * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); @@ -564,6 +569,11 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem); +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : if dictionary is created "byReference", reduce this amount by dictSize */ +ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize); + /*! ZSTD_sizeof_DDict() : * Gives the amount of memory used by a given ZSTD_DDict */ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a9dcf12e0..7ffcff26e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -404,6 +404,12 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != CNBuffSize) goto _output_error); DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize); + DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize); + } + DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); ZSTD_customMem customMem = { NULL, NULL, NULL };