@@ -19,8 +19,8 @@
- Streaming decompression - HowTo
- START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
- Advanced types
-- Compressed size functions
-- Decompressed size functions
+- Frame size functions
+- Context memory usage
- Advanced compression functions
- Advanced decompression functions
- Advanced streaming functions
@@ -244,6 +244,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are effectively same object */
+
ZSTD_CStream management functions
ZSTD_CStream* ZSTD_createCStream(void);
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -319,70 +321,101 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
ZSTD_frameParameters fParams;
} ZSTD_parameters;
+typedef struct {
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+} ZSTD_frameHeader;
+
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-Compressed size functions
+Frame size functions
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
`src` should point to the start of a ZSTD encoded frame or skippable frame
`srcSize` must be at least as large as the frame
- @return : the compressed size of the frame pointed to by `src`, suitable to pass to
- `ZSTD_decompress` or similar, or an error code if given invalid input.
+ @return : the compressed size of the frame pointed to by `src`,
+ suitable to pass to `ZSTD_decompress` or similar,
+ or an error code if given invalid input.
-Decompressed size functions
-
-unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
- `src` should point to the start of a ZSTD encoded frame
- `srcSize` must be at least as large as the frame header. A value greater than or equal
- to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
- @return : decompressed size of the frame pointed to be `src` if known, otherwise
- - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
- - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
+
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+ `src` should point to the start of a ZSTD encoded frame.
+ `srcSize` must be at least as large as the frame header.
+ A value >= `ZSTD_frameHeaderSize_max` is guaranteed to be large enough.
+ @return : - decompressed size of the frame pointed to be `src` if known
+ - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
- `src` should point the start of a series of ZSTD encoded and/or skippable frames
- `srcSize` must be the _exact_ size of this series
+
`src` should point the start of a series of ZSTD encoded and/or skippable frames
+ `srcSize` must be the _exact_ size of this series
(i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
- @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
- - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
- - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ @return : - decompressed size of all data in all successive frames
+ - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+ - if an error occurred: ZSTD_CONTENTSIZE_ERROR
- note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
- When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
- In which case, it's necessary to use streaming mode to decompress data.
- Optionally, application can still use ZSTD_decompress() while relying on implied limits.
- (For example, data may be necessarily cut into blocks <= 16 KB).
- note 2 : decompressed size is always present when compression is done with ZSTD_compress()
- note 3 : decompressed size can be very large (64-bits value),
- potentially larger than what local system can handle as a single memory segment.
- In which case, it's necessary to use streaming mode to decompress data.
- note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
- Always ensure result fits within application's authorized limits.
- Each application can set its own limits.
- note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
- read each contained frame header. This is efficient as most of the data is skipped,
- however it does mean that all frame data must be present and valid.
+ note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ In which case, it's necessary to use streaming mode to decompress data.
+ Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+ (For example, data may be necessarily cut into blocks <= 16 KB).
+ note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ note 3 : decompressed size can be very large (64-bits value),
+ potentially larger than what local system can handle as a single memory segment.
+ In which case, it's necessary to use streaming mode to decompress data.
+ note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ Always ensure result fits within application's authorized limits.
+ Each application can set its own limits.
+ note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+ read each contained frame header. This is efficient as most of the data is skipped,
+ however it does mean that all frame data must be present and valid.
+
+
+Context memory usage
+
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+ These functions give the current memory usage of selected object.
+ Object memory usage can evolve if it's re-used multiple times.
+
+
+size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateDCtxSize(void);
+ These functions make it possible to estimate memory usage
+ of a future target object, before its allocation,
+ given a set of parameters, which vary depending on target object.
+ The objective is to guide decision before allocation.
+
+
+size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
+ Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ an internal ?Dict will be created, which size is not estimated.
+ In this case, get additional size by using ZSTD_estimate?DictSize
+
+
+size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
+size_t ZSTD_estimateDDictSize(size_t dictSize);
+ Note : if dictionary is created "byReference", reduce estimation by dictSize
Advanced compression functions
-size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
- Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
- `frameContentSize` is an optional parameter, provide `0` if unknown
-
-
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
Create a ZSTD compression context using external alloc and free functions
-size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
- Gives the amount of memory used by a given ZSTD_CCtx
-
-
typedef enum {
ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -404,10 +437,6 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
Create a ZSTD_CDict using external alloc and free, and customized compression parameters
-size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
- Gives the amount of memory used by a given ZSTD_sizeof_CDict
-
-
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
@return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
`estimatedSrcSize` value is optional, select 0 if not known
@@ -451,18 +480,10 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
Note 3 : Skippable Frame Identifiers are considered valid.
-size_t ZSTD_estimateDCtxSize(void);
- Gives the potential amount of memory allocated to create a ZSTD_DCtx
-
-
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
Create a ZSTD decompression context using external alloc and free functions
-size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
- Gives the amount of memory used by a given ZSTD_DCtx
-
-
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
Create a digested dictionary, ready to start decompression operation without startup delay.
Dictionary content is simply referenced, and therefore stays in dictBuffer.
@@ -474,10 +495,6 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
Create a ZSTD_DDict using external alloc and free, optionally by reference
-size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
- Gives the amount of memory used by a given ZSTD_DDict
-
-
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
Provides the dictID stored within dictionary.
if @return == 0, the dictionary is not conformant with Zstandard specification.
@@ -505,9 +522,8 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
Advanced streaming functions
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
/**< size of CStream is variable, depending primarily on compression level */
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
@@ -524,11 +540,10 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
Advanced Streaming decompression functions
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
-size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
Buffer-less and synchronous inner streaming functions
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
@@ -631,14 +646,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
It also returns Frame Size as fparamsPtr->frameContentSize.
-typedef struct {
- unsigned long long frameContentSize;
- unsigned windowSize;
- unsigned dictID;
- unsigned checksumFlag;
-} ZSTD_frameParams;
-
-Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);
/**< doesn't consume input, see details below */
+Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);
/**< doesn't consume input, see details below */
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c08b315da..848472ec4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -79,6 +79,8 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
/*-*************************************
* Context memory management
***************************************/
+typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
+
struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */
@@ -115,6 +117,22 @@ struct ZSTD_CCtx_s {
FSE_CTable* matchlengthCTable;
FSE_CTable* litlengthCTable;
unsigned* entropyScratchSpace;
+
+ /* streaming */
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
+ char* inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ char* outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZSTD_cStreamStage streamStage;
+ U32 frameEnded;
+ U64 pledgedSrcSize;
};
ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -140,6 +158,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
{
if (cctx==NULL) return 0; /* support free on NULL */
ZSTD_free(cctx->workSpace, cctx->customMem);
+ cctx->workSpace = NULL;
+ ZSTD_freeCDict(cctx->cdictLocal);
+ cctx->cdictLocal = NULL;
+ ZSTD_free(cctx->inBuff, cctx->customMem);
+ cctx->inBuff = NULL;
+ ZSTD_free(cctx->outBuff, cctx->customMem);
+ cctx->outBuff = NULL;
ZSTD_free(cctx, cctx->customMem);
return 0; /* reserved as a potential error code in the future */
}
@@ -147,7 +172,9 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
{
if (cctx==NULL) return 0; /* support sizeof on NULL */
- return sizeof(*cctx) + cctx->workSpaceSize;
+ return sizeof(*cctx) + cctx->workSpaceSize
+ + ZSTD_sizeof_CDict(cctx->cdictLocal)
+ + cctx->outBuffSize + cctx->inBuffSize;
}
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
@@ -240,10 +267,10 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
+ entropyScratchSpace_size;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
- size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<params)) {
+ DEBUGLOG(5, "ZSTD_equivalentParams()==1 \n");
zc->fseCTables_ready = 0;
zc->hufCTable_repeatMode = HUF_repeat_none;
return ZSTD_continueCCtx(zc, params, frameContentSize);
@@ -312,6 +342,8 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
size_t const optSpace = ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optPotentialSpace : 0;
size_t const neededSpace = entropySpace + optSpace + tableSpace + tokenSpace;
if (zc->workSpaceSize < neededSpace) {
+ DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n",
+ (unsigned)zc->workSpaceSize>>10, (unsigned)neededSpace>>10);
zc->workSpaceSize = 0;
ZSTD_free(zc->workSpace, zc->customMem);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
@@ -335,6 +367,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
/* init params */
zc->params = params;
zc->blockSize = blockSize;
+ DEBUGLOG(5, "blockSize = %uK \n", (U32)blockSize>>10);
zc->frameContentSize = frameContentSize;
zc->consumedSrcSize = 0;
@@ -364,6 +397,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
/* opt parser space */
if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
+ DEBUGLOG(5, "reserving optimal parser space ");
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->seqStore.litFreq = (U32*)ptr;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<stage!=ZSTDcs_init) return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
@@ -2528,9 +2563,8 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
size_t pos;
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
- DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u \n", !params.fParams.noDictIDFlag);
- DEBUGLOG(5, "ZSTD_writeFrameHeader : dictID : %u \n", dictID);
- DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDSizeCode : %u \n", dictIDSizeCode);
+ DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u \n",
+ !params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
op[4] = frameHeaderDecriptionByte; pos=5;
@@ -2840,6 +2874,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
BYTE* op = ostart;
size_t fhSize = 0;
+ DEBUGLOG(5, "ZSTD_writeEpilogue \n");
if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */
/* special case : empty frame */
@@ -2877,12 +2912,14 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
const void* src, size_t srcSize)
{
size_t endResult;
- size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */);
+ size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize,
+ 1 /* frame mode */, 1 /* last chunk */);
if (ZSTD_isError(cSize)) return cSize;
endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
if (ZSTD_isError(endResult)) return endResult;
if (cctx->params.fParams.contentSizeFlag) { /* control src size */
- if (cctx->frameContentSize != cctx->consumedSrcSize) return ERROR(srcSize_wrong);
+ if (cctx->frameContentSize != cctx->consumedSrcSize)
+ return ERROR(srcSize_wrong);
}
return cSize + endResult;
}
@@ -2942,6 +2979,14 @@ struct ZSTD_CDict_s {
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
+/*! ZSTD_estimateCDictSize() :
+ * Estimate amount of memory that will be needed to create a dictionary with following arguments */
+size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
+{
+ cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
+ return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams);
+}
+
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support sizeof on NULL */
@@ -3084,30 +3129,6 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
* Streaming
********************************************************************/
-typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
-
-struct ZSTD_CStream_s {
- ZSTD_CCtx* cctx;
- ZSTD_CDict* cdictLocal;
- const ZSTD_CDict* cdict;
- char* inBuff;
- size_t inBuffSize;
- size_t inToCompress;
- size_t inBuffPos;
- size_t inBuffTarget;
- size_t blockSize;
- char* outBuff;
- size_t outBuffSize;
- size_t outBuffContentSize;
- size_t outBuffFlushedSize;
- ZSTD_cStreamStage stage;
- U32 checksum;
- U32 frameEnded;
- U64 pledgedSrcSize;
- ZSTD_parameters params;
- ZSTD_customMem customMem;
-}; /* typedef'd to ZSTD_CStream within "zstd.h" */
-
ZSTD_CStream* ZSTD_createCStream(void)
{
return ZSTD_createCStream_advanced(defaultCustomMem);
@@ -3115,35 +3136,24 @@ ZSTD_CStream* ZSTD_createCStream(void)
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
{
- ZSTD_CStream* zcs;
-
- if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
- if (!customMem.customAlloc || !customMem.customFree) return NULL;
-
- zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
- if (zcs==NULL) return NULL;
- memset(zcs, 0, sizeof(ZSTD_CStream));
- memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
- zcs->cctx = ZSTD_createCCtx_advanced(customMem);
- if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; }
- return zcs;
+ /* CStream and CCtx are now same object */
+ return ZSTD_createCCtx_advanced(customMem);
}
size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
{
- if (zcs==NULL) return 0; /* support free on NULL */
- { ZSTD_customMem const cMem = zcs->customMem;
- ZSTD_freeCCtx(zcs->cctx);
- zcs->cctx = NULL;
- ZSTD_freeCDict(zcs->cdictLocal);
- zcs->cdictLocal = NULL;
- ZSTD_free(zcs->inBuff, cMem);
- zcs->inBuff = NULL;
- ZSTD_free(zcs->outBuff, cMem);
- zcs->outBuff = NULL;
- ZSTD_free(zcs, cMem);
- return 0;
- }
+ return ZSTD_freeCCtx(zcs); /* same object */
+}
+
+size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
+{
+ size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
+ size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
+ size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
+ size_t const streamingSize = inBuffSize + outBuffSize;
+
+ return CCtxSize + streamingSize;
}
@@ -3156,20 +3166,20 @@ size_t ZSTD_CStreamOutSize(void)
return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
}
-static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
- if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize))
- else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
+ if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
+ else CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize));
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
- zcs->stage = zcss_load;
+ zcs->streamStage = zcss_load;
zcs->frameEnded = 0;
zcs->pledgedSrcSize = pledgedSrcSize;
return 0; /* ready to go */
@@ -3178,9 +3188,10 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
- zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+ ZSTD_parameters params = zcs->params;
+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
DEBUGLOG(5, "ZSTD_resetCStream : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
- return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
+ return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
}
/* ZSTD_initCStream_internal() :
@@ -3191,32 +3202,29 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
unsigned long long pledgedSrcSize)
{
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+ zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
/* allocate buffers */
- { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
+ { size_t const neededInBuffSize = ((size_t)1 << params.cParams.windowLog) + zcs->blockSize;
if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = 0;
ZSTD_free(zcs->inBuff, zcs->customMem);
- zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem);
+ zcs->inBuff = (char*)ZSTD_malloc(neededInBuffSize, zcs->customMem);
if (zcs->inBuff == NULL) return ERROR(memory_allocation);
zcs->inBuffSize = neededInBuffSize;
}
- zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
}
if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
size_t const outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
zcs->outBuffSize = 0;
ZSTD_free(zcs->outBuff, zcs->customMem);
- zcs->outBuff = (char*) ZSTD_malloc(outBuffSize, zcs->customMem);
+ zcs->outBuff = (char*)ZSTD_malloc(outBuffSize, zcs->customMem);
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
zcs->outBuffSize = outBuffSize;
}
- zcs->checksum = params.fParams.checksumFlag > 0;
- zcs->params = params;
-
DEBUGLOG(5, "ZSTD_initCStream_stage2 : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
- return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
+ return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
}
/* ZSTD_initCStream_usingCDict_advanced() :
@@ -3261,7 +3269,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
CHECK_F( ZSTD_checkCParams(params.cParams) );
- DEBUGLOG(5, "ZSTD_initCStream_advanced : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
+ DEBUGLOG(5, "ZSTD_initCStream_advanced : pledgedSrcSize == %u \n", (U32)pledgedSrcSize);
+ DEBUGLOG(5, "wlog %u \n", params.cParams.windowLog);
return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
}
@@ -3286,8 +3295,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{
- if (zcs==NULL) return 0; /* support sizeof on NULL */
- return sizeof(*zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
+ return ZSTD_sizeof_CCtx(zcs); /* same object */
}
/*====== Compression ======*/
@@ -3314,8 +3322,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
char* const oend = ostart + *dstCapacityPtr;
char* op = ostart;
+ DEBUGLOG(5, "ZSTD_compressStream_generic \n");
while (someMoreWork) {
- switch(zcs->stage)
+ switch(zcs->streamStage)
{
case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
@@ -3323,12 +3332,14 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
/* complete inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
+ DEBUGLOG(5, "loading %u/%u \n", (U32)loaded, (U32)toLoad);
zcs->inBuffPos += loaded;
ip += loaded;
if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) {
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */
} }
/* compress current block (note : this stage cannot be stopped in the middle) */
+ DEBUGLOG(5, "stream compression stage (flush==%u)\n", flush);
{ void* cDst;
size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
@@ -3338,29 +3349,33 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ?
- ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
- ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
+ ZSTD_compressEnd(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
+ ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
+ DEBUGLOG(5, "cSize = %u \n", (U32)cSize);
if (flush == zsf_end) zcs->frameEnded = 1;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize)
- zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
+ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffTarget == blockSize <= inBuffSize */
+ assert(zcs->inBuffTarget <= zcs->inBuffSize);
zcs->inToCompress = zcs->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */
zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0;
- zcs->stage = zcss_flush; /* pass-through to flush stage */
+ zcs->streamStage = zcss_flush; /* pass-through to flush stage */
}
case zcss_flush:
+ DEBUGLOG(5, "flush stage \n");
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+ DEBUGLOG(5, "toFlush: %u ; flushed: %u \n", (U32)toFlush, (U32)flushed);
op += flushed;
zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
- zcs->stage = zcss_load;
+ zcs->streamStage = zcss_load;
break;
}
@@ -3419,7 +3434,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
- if (zcs->stage != zcss_final) {
+ DEBUGLOG(5, "ZSTD_endStream (dstCapacity : %u) \n", (U32)(oend-op));
+ if (zcs->streamStage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos;
@@ -3429,13 +3445,14 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
op += sizeWritten;
if (remainingToFlush) {
output->pos += sizeWritten;
- return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
+ return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */
+ + ((zcs->params.fParams.checksumFlag > 0) * 4) /* optional 32-bits checksum */;
}
/* create epilogue */
- zcs->stage = zcss_final;
+ zcs->streamStage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 :
/* write epilogue, including final empty block, into outBuff */
- ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0);
+ ZSTD_compressEnd(zcs, zcs->outBuff, zcs->outBuffSize, NULL, 0);
if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize;
}
@@ -3445,7 +3462,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
op += flushed;
zcs->outBuffFlushedSize += flushed;
output->pos += op-ostart;
- if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */
+ if (toFlush==flushed) zcs->streamStage = zcss_init; /* end reached */
return toFlush - flushed;
}
}
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 910f9ab78..379842b57 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -105,7 +105,7 @@ struct ZSTD_DCtx_s
const void* vBase; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
- ZSTD_frameParams fParams;
+ ZSTD_frameHeader fParams;
blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage;
U32 litEntropy;
@@ -219,22 +219,23 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
}
-/** ZSTD_getFrameParams() :
+/** ZSTD_getFrameHeader() :
* decode Frame Header, or require larger `srcSize`.
-* @return : 0, `fparamsPtr` is correctly filled,
+* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, result is expected `srcSize`,
* or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize)
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
-
if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
+
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ /* skippable frame */
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
- memset(fparamsPtr, 0, sizeof(*fparamsPtr));
- fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
- fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
+ memset(zfhPtr, 0, sizeof(*zfhPtr));
+ zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
+ zfhPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
return 0;
}
return ERROR(prefix_unknown);
@@ -281,10 +282,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
}
if (!windowSize) windowSize = (U32)frameContentSize;
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
- fparamsPtr->frameContentSize = frameContentSize;
- fparamsPtr->windowSize = windowSize;
- fparamsPtr->dictID = dictID;
- fparamsPtr->checksumFlag = checksumFlag;
+ zfhPtr->frameContentSize = frameContentSize;
+ zfhPtr->windowSize = windowSize;
+ zfhPtr->dictID = dictID;
+ zfhPtr->checksumFlag = checksumFlag;
}
return 0;
}
@@ -302,9 +303,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
}
#endif
- {
- ZSTD_frameParams fParams;
- if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
+ { ZSTD_frameHeader fParams;
+ if (ZSTD_getFrameHeader(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
if (fParams.windowSize == 0) {
/* Either skippable or empty frame, size == 0 either way */
return 0;
@@ -389,7 +389,7 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
{
- size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
+ size_t const result = ZSTD_getFrameHeader(&(dctx->fParams), src, headerSize);
if (ZSTD_isError(result)) return result; /* invalid header */
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
@@ -1364,13 +1364,13 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
const BYTE* ip = (const BYTE*)src;
const BYTE* const ipstart = ip;
size_t remainingSize = srcSize;
- ZSTD_frameParams fParams;
+ ZSTD_frameHeader fParams;
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
if (ZSTD_isError(headerSize)) return headerSize;
/* Frame Header */
- { size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
+ { size_t const ret = ZSTD_getFrameHeader(&fParams, ip, remainingSize);
if (ZSTD_isError(ret)) return ret;
if (ret > 0) return ERROR(srcSize_wrong);
}
@@ -1969,6 +1969,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
}
}
+/*! ZSTD_estimateDDictSize() :
+ * Estimate amount of memory that will be needed to create a dictionary for decompression.
+ * Note : if dictionary is created "byReference", reduce this amount by dictSize */
+size_t ZSTD_estimateDDictSize(size_t dictSize)
+{
+ return dictSize + sizeof(ZSTD_DDict);
+}
+
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
@@ -2008,11 +2016,11 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
* Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to use
- * ZSTD_getFrameParams(), which will provide a more precise error code. */
+ * ZSTD_getFrameHeader(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
{
- ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 };
- size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
+ ZSTD_frameHeader zfp = { 0 , 0 , 0 , 0 };
+ size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0;
return zfp.dictID;
}
@@ -2045,7 +2053,7 @@ struct ZSTD_DStream_s {
ZSTD_DCtx* dctx;
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict;
- ZSTD_frameParams fParams;
+ ZSTD_frameHeader fParams;
ZSTD_dStreamStage stage;
char* inBuff;
size_t inBuffSize;
@@ -2175,6 +2183,15 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
+ zds->inBuffSize + zds->outBuffSize;
}
+size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
+{
+ size_t const windowSize = fHeader.windowSize;
+ size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ size_t const inBuffSize = blockSize; /* no block can be larger */
+ size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
+ return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
+}
+
/* ***** Decompression ***** */
@@ -2209,7 +2226,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* fall-through */
case zdss_loadHeader :
- { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
+ { size_t const hSize = ZSTD_getFrameHeader(&zds->fParams, zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize))
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
diff --git a/lib/zstd.h b/lib/zstd.h
index f8050c136..c956860ab 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -55,7 +55,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
-#define ZSTD_VERSION_MINOR 2
+#define ZSTD_VERSION_MINOR 3
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
@@ -281,7 +281,8 @@ typedef struct ZSTD_outBuffer_s {
*
* *******************************************************************/
-typedef struct ZSTD_CStream_s ZSTD_CStream;
+typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are effectively same object */
+ /* Continue due distinghish them for compatibility with versions <= v1.2.0 */
/*===== ZSTD_CStream management functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -349,9 +350,6 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
-#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
-
#define ZSTD_WINDOWLOG_MAX_32 27
#define ZSTD_WINDOWLOG_MAX_64 27
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
@@ -400,76 +398,109 @@ typedef struct {
ZSTD_frameParameters fParams;
} ZSTD_parameters;
+typedef struct {
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+} ZSTD_frameHeader;
+
/*= Custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
/***************************************
-* Compressed size functions
+* Frame size functions
***************************************/
/*! ZSTD_findFrameCompressedSize() :
* `src` should point to the start of a ZSTD encoded frame or skippable frame
* `srcSize` must be at least as large as the frame
- * @return : the compressed size of the frame pointed to by `src`, suitable to pass to
- * `ZSTD_decompress` or similar, or an error code if given invalid input. */
+ * @return : the compressed size of the frame pointed to by `src`,
+ * suitable to pass to `ZSTD_decompress` or similar,
+ * or an error code if given invalid input. */
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-/***************************************
-* Decompressed size functions
-***************************************/
/*! ZSTD_getFrameContentSize() :
-* `src` should point to the start of a ZSTD encoded frame
-* `srcSize` must be at least as large as the frame header. A value greater than or equal
-* to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
-* @return : decompressed size of the frame pointed to be `src` if known, otherwise
-* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
-* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+ * `src` should point to the start of a ZSTD encoded frame.
+ * `srcSize` must be at least as large as the frame header.
+ * A value >= `ZSTD_frameHeaderSize_max` is guaranteed to be large enough.
+ * @return : - decompressed size of the frame pointed to be `src` if known
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
+#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
/*! ZSTD_findDecompressedSize() :
-* `src` should point the start of a series of ZSTD encoded and/or skippable frames
-* `srcSize` must be the _exact_ size of this series
-* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
-* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
-* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
-* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
-*
-* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
-* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
-* In which case, it's necessary to use streaming mode to decompress data.
-* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
-* (For example, data may be necessarily cut into blocks <= 16 KB).
-* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
-* note 3 : decompressed size can be very large (64-bits value),
-* potentially larger than what local system can handle as a single memory segment.
-* In which case, it's necessary to use streaming mode to decompress data.
-* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
-* Always ensure result fits within application's authorized limits.
-* Each application can set its own limits.
-* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
-* read each contained frame header. This is efficient as most of the data is skipped,
-* however it does mean that all frame data must be present and valid. */
+ * `src` should point the start of a series of ZSTD encoded and/or skippable frames
+ * `srcSize` must be the _exact_ size of this series
+ * (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
+ * @return : - decompressed size of all data in all successive frames
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+ *
+ * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+ * (For example, data may be necessarily cut into blocks <= 16 KB).
+ * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+ * note 3 : decompressed size can be very large (64-bits value),
+ * potentially larger than what local system can handle as a single memory segment.
+ * In which case, it's necessary to use streaming mode to decompress data.
+ * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+ * Always ensure result fits within application's authorized limits.
+ * Each application can set its own limits.
+ * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+ * read each contained frame header. This is efficient as most of the data is skipped,
+ * however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+/***************************************
+* Context memory usage
+***************************************/
+
+/*! ZSTD_sizeof_*() :
+ * These functions give the current memory usage of selected object.
+ * Object memory usage can evolve if it's re-used multiple times. */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_estimate*() :
+ * These functions make it possible to estimate memory usage
+ * of a future target object, before its allocation,
+ * given a set of parameters, which vary depending on target object.
+ * The objective is to guide decision before allocation. */
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
+
+/*! ZSTD_estimate?StreamSize() :
+ * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+ * an internal ?Dict will be created, which size is not estimated.
+ * In this case, get additional size by using ZSTD_estimate?DictSize */
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
+
+/*! ZSTD_estimate?DictSize() :
+ * Note : if dictionary is created "byReference", reduce estimation by dictSize */
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
+ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
+
+
/***************************************
* Advanced compression functions
***************************************/
-/*! ZSTD_estimateCCtxSize() :
- * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
- * `frameContentSize` is an optional parameter, provide `0` if unknown */
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
-
/*! ZSTD_createCCtx_advanced() :
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
-/*! ZSTD_sizeofCCtx() :
- * Gives the amount of memory used by a given ZSTD_CCtx */
-ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-
typedef enum {
ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -490,10 +521,6 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
-/*! ZSTD_sizeof_CDict() :
- * Gives the amount of memory used by a given ZSTD_sizeof_CDict */
-ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
* `estimatedSrcSize` value is optional, select 0 if not known */
@@ -538,18 +565,10 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
* Note 3 : Skippable Frame Identifiers are considered valid. */
ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
-/*! ZSTD_estimateDCtxSize() :
- * Gives the potential amount of memory allocated to create a ZSTD_DCtx */
-ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
-
/*! ZSTD_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
-/*! ZSTD_sizeof_DCtx() :
- * Gives the amount of memory used by a given ZSTD_DCtx */
-ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
@@ -561,10 +580,6 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
-/*! ZSTD_sizeof_DDict() :
- * Gives the amount of memory used by a given ZSTD_DDict */
-ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
@@ -596,9 +611,8 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< size of CStream is variable, depending primarily on compression level */
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
@@ -617,11 +631,10 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
/*===== Advanced Streaming decompression functions =====*/
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
-ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
/*********************************************************************
@@ -736,15 +749,8 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
It also returns Frame Size as fparamsPtr->frameContentSize.
*/
-typedef struct {
- unsigned long long frameContentSize;
- unsigned windowSize;
- unsigned dictID;
- unsigned checksumFlag;
-} ZSTD_frameParams;
-
/*===== Buffer-less streaming decompression functions =====*/
-ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
+ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 38cf22fa7..13323aec1 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -297,10 +297,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 31: /* ZSTD_decodeLiteralsBlock */
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
{ blockProperties_t bp;
- ZSTD_frameParams zfp;
+ ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
- frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
+ frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
@@ -315,13 +315,13 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 32: /* ZSTD_decodeSeqHeaders */
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
{ blockProperties_t bp;
- ZSTD_frameParams zfp;
+ ZSTD_frameHeader zfp;
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
- frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
+ frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a9dcf12e0..4b9cd97a5 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -339,8 +339,8 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize),
cSize = r);
- { ZSTD_frameParams fp;
- if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
+ { ZSTD_frameHeader fp;
+ if (ZSTD_getFrameHeader(&fp, compressedBuffer, cSize)) goto _output_error;
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
@@ -404,6 +404,12 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
+ DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
+ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+ size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
+ DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
+ }
+
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem customMem = { NULL, NULL, NULL };
@@ -856,8 +862,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
}
/* frame header decompression test */
- { ZSTD_frameParams dParams;
- size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);
+ { ZSTD_frameHeader dParams;
+ size_t const check = ZSTD_getFrameHeader(&dParams, cBuffer, cSize);
CHECK(ZSTD_isError(check), "Frame Parameters extraction failed");
CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
}
diff --git a/tests/symbols.c b/tests/symbols.c
index 7dacfc058..ade3aa02c 100644
--- a/tests/symbols.c
+++ b/tests/symbols.c
@@ -88,7 +88,7 @@ static const void *symbols[] = {
&ZSTD_copyCCtx,
&ZSTD_compressContinue,
&ZSTD_compressEnd,
- &ZSTD_getFrameParams,
+ &ZSTD_getFrameHeader,
&ZSTD_decompressBegin,
&ZSTD_decompressBegin_usingDict,
&ZSTD_copyDCtx,
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0e09e1856..5c166bfb8 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -53,13 +53,15 @@ static const U32 prime32 = 2654435761U;
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
+ DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stderr); }
static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stderr); } }
+ if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;
@@ -155,12 +157,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
void* decodedBuffer = malloc(decodedBufferSize);
size_t cSize;
int testResult = 0;
- U32 testNb=0;
+ U32 testNb = 1;
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
ZSTD_inBuffer inBuff, inBuff2;
ZSTD_outBuffer outBuff;
buffer_t dictionary = g_nullBuffer;
+ size_t const dictSize = 128 KB;
unsigned dictID = 0;
/* Create compressible test buffer */
@@ -186,7 +189,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
- ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
+ ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1);
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@@ -201,7 +204,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
cSize += outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
- DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
+ /* context size functions */
+ DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
+ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
+ size_t const s = ZSTD_estimateCStreamSize(cParams)
+ + ZSTD_estimateCDictSize(cParams, dictSize); /* uses ZSTD_initCStream_usingDict() */
+ if (ZSTD_isError(s)) goto _output_error;
+ DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
+ }
+
+ DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_CStream(zc);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
@@ -219,7 +231,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* skippable frame test */
DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
- ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+ ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
@@ -234,7 +246,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic decompression test */
inBuff2 = inBuff;
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
- ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+ ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
{ size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000); /* large limit */
if (ZSTD_isError(r)) goto _output_error; }
{ size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
@@ -260,7 +272,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
} }
DISPLAYLEVEL(3, "OK \n");
- DISPLAYLEVEL(3, "test%3i : check DStream size : ", testNb++);
+ /* context size functions */
+ DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
+ { ZSTD_frameHeader fhi;
+ const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
+ size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
+ if (gfhError!=0) goto _output_error;
+ DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
+ { size_t const s = ZSTD_estimateDStreamSize(fhi)
+ + ZSTD_estimateDDictSize(dictSize); /* uses ZSTD_initDStream_usingDict() */
+ if (ZSTD_isError(s)) goto _output_error;
+ DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
+ } }
+
+ DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_DStream(zd);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
@@ -270,7 +295,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
{ /* skippable frame */
size_t r = 1;
- ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+ ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
inBuff.src = compressedBuffer;
outBuff.dst = decodedBuffer;
inBuff.pos = 0;
@@ -282,7 +307,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (ZSTD_isError(r)) goto _output_error;
}
/* normal frame */
- ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+ ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
r=1;
while (r) {
inBuff.size = inBuff.pos + 1;
@@ -344,6 +369,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (zc==NULL) goto _output_error; /* memory allocation issue */
/* use 1 */
{ size_t const inSize = 513;
+ DISPLAYLEVEL(5, "use1 ");
ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
inBuff.src = CNBuffer;
inBuff.size = inSize;
@@ -351,14 +377,17 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = ZSTD_compressBound(inSize);
outBuff.pos = 0;
+ DISPLAYLEVEL(5, "compress1 ");
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ DISPLAYLEVEL(5, "end1 ");
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
}
/* use 2 */
{ size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
+ DISPLAYLEVEL(5, "use2 ");
ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
inBuff.src = CNBuffer;
inBuff.size = inSize;
@@ -366,9 +395,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = ZSTD_compressBound(inSize);
outBuff.pos = 0;
+ DISPLAYLEVEL(5, "compress2 ");
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ DISPLAYLEVEL(5, "end2 ");
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
}
@@ -435,7 +466,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Memory restriction */
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
- ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+ ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
{ size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000); /* too small limit */
if (ZSTD_isError(r)) goto _output_error; }
inBuff.src = compressedBuffer;
@@ -770,7 +801,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
outBuff.size = outBuff.pos + adjustedDstSize;
remainingToFlush = ZSTD_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
- CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
+ CHECK (enoughDstSize && remainingToFlush,
+ "ZSTD_endStream() not fully flushed (%u remaining), but enough space available (%u)",
+ (U32)remainingToFlush, (U32)adjustedDstSize);
} }
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;