diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index b4720adac..7ef86a23d 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -631,10 +631,10 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);
/**< same as ZSTD_initStaticCCtx() */
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
@@ -642,9 +642,11 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
start a new compression job, using same parameters from previous job.
This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
Note that zcs must be init at least once before using ZSTD_resetCStream().
- pledgedSrcSize==0 means "srcSize unknown".
+ If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
- @return : 0, or an error code (which can be tested using ZSTD_isError())
+ For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
+ @return : 0, or an error code (which can be tested using ZSTD_isError())
Advanced Streaming decompression functions
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
@@ -693,10 +695,10 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds);
/**< re-use decompression para
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations.
@@ -848,11 +850,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
* Special: value 0 means "do not change strategy". */
/* frame parameters */
- ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1)
- * note that content size must be known at the beginning,
- * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
+ ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+ * Content size must be known at the beginning of compression,
+ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
- ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+ ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
/* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
@@ -913,7 +915,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Note 1 : 0 means zero, empty.
In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+ ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
Note 2 : If all data is provided and consumed in a single round,
this value is overriden by srcSize instead.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 43e5f5461..635cc665c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -78,6 +78,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
if (!cctx) return NULL;
cctx->customMem = customMem;
cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
+ cctx->requestedParams.fParams.contentSizeFlag = 1;
ZSTD_STATIC_ASSERT(zcss_init==0);
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
return cctx;
@@ -391,7 +392,7 @@ size_t ZSTD_CCtxParam_setParameter(
case ZSTD_p_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */
- DEBUGLOG(5, "set content size flag = %u", (value>0));
+ DEBUGLOG(4, "set content size flag = %u", (value>0));
params->fParams.contentSizeFlag = value > 0;
return 0;
@@ -401,7 +402,7 @@ size_t ZSTD_CCtxParam_setParameter(
return 0;
case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
- DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
+ DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
params->fParams.noDictIDFlag = (value == 0);
return 0;
@@ -530,7 +531,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
cctx->cdict = NULL;
} else {
ZSTD_compressionParameters const cParams =
- ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, 0, dictSize);
+ ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize,
dictLoadMethod, dictMode,
@@ -821,9 +822,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_compResetPolicy_e const crp,
ZSTD_buffered_policy_e const zbuff)
{
- DEBUGLOG(4, "ZSTD_resetCCtx_internal");
+ DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u", (U32)pledgedSrcSize);
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
- DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize);
if (crp == ZSTDcrp_continue) {
if (ZSTD_equivalentParams(params, zc->appliedParams)) {
@@ -1002,7 +1002,7 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
* The "context", in this case, refers to the hash and chain tables, entropy
* tables, and dictionary offsets.
* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
- * pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1
+ * pledgedSrcSize=0 means "empty".
* @return : 0, or an error code */
static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
const ZSTD_CCtx* srcCCtx,
@@ -1059,7 +1059,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
- fParams.contentSizeFlag = pledgedSrcSize>0;
+ if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff);
}
@@ -2070,6 +2071,8 @@ size_t ZSTD_compressBegin_advanced_internal(
ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{
+ DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal");
+ DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag);
/* compression parameters verification and optimization */
CHECK_F( ZSTD_checkCParams(params.cParams) );
return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
@@ -2096,7 +2099,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
- cctxParams, 0, ZSTDb_not_buffered);
+ cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
}
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
@@ -2298,7 +2301,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{
- DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
+ DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@@ -2373,9 +2376,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
+ cctxSize;
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
void* ptr;
- DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+ DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
- DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
+ DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
(U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
if (workspaceSize < neededSize) return NULL;
@@ -2411,7 +2414,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
{ ZSTD_CCtx_params params = cctx->requestedParams;
params.cParams = ZSTD_getCParamsFromCDict(cdict);
params.fParams = fParams;
- DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
+ DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto,
cdict,
@@ -2426,7 +2429,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
- DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
+ DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
}
@@ -2515,12 +2518,15 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
return 0; /* ready to go */
}
+/* ZSTD_resetCStream():
+ * pledgedSrcSize == 0 means "unknown" */
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
ZSTD_CCtx_params params = zcs->requestedParams;
- params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+ DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize);
+ if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+ params.fParams.contentSizeFlag = 1;
params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0);
- DEBUGLOG(4, "ZSTD_resetCStream");
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
}
@@ -2587,13 +2593,20 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */
}
+/* ZSTD_initCStream_advanced() :
+ * pledgedSrcSize : if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
+ DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
+ (U32)pledgedSrcSize, params.fParams.contentSizeFlag);
CHECK_F( ZSTD_checkCParams(params.cParams) );
+ if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0))
+ pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize);
}
@@ -2602,7 +2615,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
- return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0);
+ return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
}
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
@@ -2816,18 +2829,18 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
}
DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads);
CHECK_F( ZSTDMT_initCStream_internal(
- cctx->mtctx,
- prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
- cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+ cctx->mtctx,
+ prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
+ cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
cctx->streamStage = zcss_load;
cctx->appliedParams.nbThreads = params.nbThreads;
} else
#endif
- {
- CHECK_F( ZSTD_resetCStream_internal(
+ { CHECK_F( ZSTD_resetCStream_internal(
cctx, prefixDict.dict, prefixDict.dictSize,
prefixDict.dictMode, cctx->cdict, params,
cctx->pledgedSrcSizePlusOne-1) );
+ assert(cctx->streamStage == zcss_load);
} }
/* compression stage */
@@ -3053,5 +3066,6 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
memset(¶ms, 0, sizeof(params));
params.cParams = cParams;
+ params.fParams.contentSizeFlag = 1;
return params;
}
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 7831cd3bd..af7be3167 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -327,6 +327,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t dstBuff = job->dstBuff;
+ DEBUGLOG(5, "ZSTDMT_compressChunk")
DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
@@ -344,22 +345,22 @@ void ZSTDMT_compressChunk(void* jobDescription)
job->dstBuff = dstBuff;
}
- if (job->cdict) { /* should only happen for first segment */
+ if (job->cdict) {
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
- DEBUGLOG(5, "using CDict");
+ DEBUGLOG(4, "ZSTDMT_compressChunk, using CDict");
+ assert(job->firstChunk); /* should only happen for first segment */
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */
- if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */
- { ZSTD_CCtx_params jobParams = job->params;
- size_t const forceWindowError =
- ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
- /* Force loading dictionary in "content-only" mode (no header analysis) */
- size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize);
- if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
- job->cSize = initError;
- goto _endJob;
- }
- } }
+ ZSTD_CCtx_params jobParams = job->params;
+ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
+ U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
+ /* load dictionary in "content-only" mode (no header analysis) */
+ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
+ if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
+ job->cSize = initError;
+ goto _endJob;
+ }
+ }
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
@@ -367,7 +368,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
}
DEBUGLOG(5, "Compressing : ");
- DEBUG_PRINTHEX(4, job->srcStart, 12);
+ DEBUG_PRINTHEX(6, job->srcStart, 12);
job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
@@ -610,6 +611,7 @@ static size_t ZSTDMT_compress_advanced_internal(
assert(jobParams.nbThreads == 0);
assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
+ DEBUGLOG(4, "ZSTDMT_compress_advanced_internal");
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
if (nbChunks==1) { /* fallback to single-thread mode */
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
@@ -641,7 +643,7 @@ static size_t ZSTDMT_compress_advanced_internal(
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
mtctx->jobs[u].dictSize = dictSize;
mtctx->jobs[u].srcSize = chunkSize;
- mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL;
+ mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
mtctx->jobs[u].fullFrameSize = srcSize;
mtctx->jobs[u].params = jobParams;
/* do not calculate checksum within sections, but write it in header for first section */
@@ -777,7 +779,6 @@ size_t ZSTDMT_initCStream_internal(
zcs->params = params;
zcs->frameContentSize = pledgedSrcSize;
if (dict) {
- DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
@@ -785,7 +786,6 @@ size_t ZSTDMT_initCStream_internal(
zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
- DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = NULL;
zcs->cdict = cdict;
@@ -838,9 +838,12 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
/* ZSTDMT_resetCStream() :
- * pledgedSrcSize is optional and can be zero == unknown */
+ * pledgedSrcSize can be zero == unknown (for the time being)
+ * prefer using ZSTD_CONTENTSIZE_UNKNOWN,
+ * as `0` might mean "empty" in the future */
size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
{
+ if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
if (zcs->params.nbThreads==1)
return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params,
@@ -852,7 +855,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
ZSTD_CCtx_params cctxParams = zcs->params;
cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams;
- return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0);
+ return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
}
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 8c59c684f..a7ce7c7dd 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
/* === Streaming functions === */
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
-ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
diff --git a/lib/zstd.h b/lib/zstd.h
index 2194a3b23..4681d3f2a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -731,10 +731,10 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
-ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
@@ -742,9 +742,11 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
* start a new compression job, using same parameters from previous job.
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
* Note that zcs must be init at least once before using ZSTD_resetCStream().
- * pledgedSrcSize==0 means "srcSize unknown".
+ * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
- * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ * but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
@@ -800,10 +802,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress
/*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -1000,11 +1002,11 @@ typedef enum {
* Special: value 0 means "do not change strategy". */
/* frame parameters */
- ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1)
- * note that content size must be known at the beginning,
- * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
+ ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+ * Content size must be known at the beginning of compression,
+ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
- ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+ ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
/* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
@@ -1066,7 +1068,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Note 1 : 0 means zero, empty.
* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- * Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+ * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
* Note 2 : If all data is provided and consumed in a single round,
* this value is overriden by srcSize instead. */
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
diff --git a/programs/Makefile b/programs/Makefile
index cbb67e9fe..1b21281b4 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -37,8 +37,8 @@ endif
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder \
- -DZSTD_NEWAPI \
- -DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
+ -DXXH_NAMESPACE=ZSTD_ \
+ -DZSTD_NEWAPI
CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
diff --git a/programs/fileio.c b/programs/fileio.c
index 70158f930..03e653893 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -439,9 +439,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
#ifdef ZSTD_NEWAPI
{ /* frame parameters */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
- (void)srcSize;
/* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
/* long distance matching */
@@ -467,7 +467,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */
+ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
}
#elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
@@ -815,11 +817,11 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* init */
#ifdef ZSTD_NEWAPI
if (fileSize!=0) /* when src is stdin, fileSize==0, but is effectively unknown */
- ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); /* note : fileSize==0 means "empty" */
+ ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
#elif defined(ZSTD_MULTITHREAD)
- CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */
+ CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) );
#else
- CHECK( ZSTD_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */
+ CHECK( ZSTD_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) );
#endif
/* Main compression loop */
@@ -1005,7 +1007,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
size_t dfnSize = FNSPACE;
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
- U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
+ U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : UTIL_getFileSize(inFileNamesTable[0]) ;
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
/* init */
@@ -1031,9 +1033,9 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
free(dstFileName);
dfnSize = ifnSize + 20;
dstFileName = (char*)malloc(dfnSize);
- if (!dstFileName)
+ if (!dstFileName) {
EXM_THROW(30, "zstd: %s", strerror(errno));
- }
+ } }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
@@ -1926,7 +1928,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
return detectError;
}
-static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
+static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
double const windowSizeUnit = (double)info->windowSize / unit;
@@ -1949,8 +1951,10 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
checkString, inFileName);
}
} else {
+ DISPLAYOUT("%s \n", inFileName);
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
- DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
+ if (info->numSkippableFrames)
+ DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n",
windowSizeUnit, unitStr,
(unsigned long long)info->windowSize);
@@ -1982,7 +1986,6 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
}
static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
- /* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
{ int const error = getFileInfo(&info, inFileName);
@@ -2022,7 +2025,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
for (u=0; u 1 && displayLevel <= 2) {
+ if (numFiles > 1 && displayLevel <= 2) { /* display total */
unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)total.compressedSize / unit;
@@ -2042,8 +2045,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
total.numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, total.nbFiles);
- }
- }
+ } }
return error;
}
}
diff --git a/tests/fullbench.c b/tests/fullbench.c
index db00ce217..3f497b034 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -233,7 +233,7 @@ static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
(void)buff2;
- ZSTD_compressBegin(g_zcc, 1);
+ ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
diff --git a/tests/playTests.sh b/tests/playTests.sh
index bc021648c..f008d8998 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -91,7 +91,7 @@ else
hasMT="true"
fi
-$ECHO "\n**** simple tests **** "
+$ECHO "\n===> simple tests "
./datagen > tmp
$ECHO "test : basic compression "
@@ -166,7 +166,7 @@ $ZSTD -f tmp && die "tmp not present : should have failed"
test ! -f tmp.zst # tmp.zst should not be created
-$ECHO "\n**** Advanced compression parameters **** "
+$ECHO "\n===> Advanced compression parameters "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
@@ -180,14 +180,14 @@ roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7
roundTripTest -g512K 19
-$ECHO "\n**** Pass-Through mode **** "
+$ECHO "\n===> Pass-Through mode "
$ECHO "Hello world 1!" | $ZSTD -df
$ECHO "Hello world 2!" | $ZSTD -dcf
$ECHO "Hello world 3!" > tmp1
$ZSTD -dcf tmp1
-$ECHO "\n**** frame concatenation **** "
+$ECHO "\n===> frame concatenation "
$ECHO "hello " > hello.tmp
$ECHO "world!" > world.tmp
@@ -218,7 +218,7 @@ $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
-$ECHO "\n**** symbolic link test **** "
+$ECHO "\n===> symbolic link test "
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
$ECHO "hello world" > hello.tmp
@@ -233,7 +233,7 @@ rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
fi
-$ECHO "\n**** test sparse file support **** "
+$ECHO "\n===> test sparse file support "
./datagen -g5M -P100 > tmpSparse
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
@@ -260,7 +260,7 @@ $DIFF tmpSparse2M tmpSparseRegenerated
rm tmpSparse*
-$ECHO "\n**** multiple files tests **** "
+$ECHO "\n===> multiple files tests "
./datagen -s1 > tmp1 2> $INTOVOID
./datagen -s2 -g100K > tmp2 2> $INTOVOID
@@ -283,7 +283,7 @@ $ECHO "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
-$ECHO "\n**** dictionary tests **** "
+$ECHO "\n===> dictionary tests "
$ECHO "- test with raw dict (content only) "
./datagen > tmpDict
@@ -339,7 +339,7 @@ $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is
rm tmp*
-$ECHO "\n**** cover dictionary tests **** "
+$ECHO "\n===> cover dictionary builder : advanced options "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
@@ -359,7 +359,7 @@ $ECHO "- Create dictionary with size limit"
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
-$ECHO "\n**** legacy dictionary tests **** "
+$ECHO "\n===> legacy dictionary builder "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
@@ -380,7 +380,7 @@ $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
-$ECHO "\n**** integrity tests **** "
+$ECHO "\n===> integrity tests "
$ECHO "test one file (tmp1.zst) "
./datagen > tmp1
@@ -405,13 +405,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
-$ECHO "\n**** golden files tests **** "
+$ECHO "\n===> golden files tests "
$ZSTD -t -r files
$ZSTD -c -r files | $ZSTD -t
-$ECHO "\n**** benchmark mode tests **** "
+$ECHO "\n===> benchmark mode tests "
$ECHO "bench one file"
./datagen > tmp1
@@ -422,7 +422,7 @@ $ECHO "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1
-$ECHO "\n**** gzip compatibility tests **** "
+$ECHO "\n===> gzip compatibility tests "
GZIPMODE=1
$ZSTD --format=gzip -V || GZIPMODE=0
@@ -445,7 +445,7 @@ else
fi
-$ECHO "\n**** gzip frame tests **** "
+$ECHO "\n===> gzip frame tests "
if [ $GZIPMODE -eq 1 ]; then
./datagen > tmp
@@ -459,7 +459,7 @@ else
fi
-$ECHO "\n**** xz compatibility tests **** "
+$ECHO "\n===> xz compatibility tests "
LZMAMODE=1
$ZSTD --format=xz -V || LZMAMODE=0
@@ -505,7 +505,7 @@ else
fi
-$ECHO "\n**** xz frame tests **** "
+$ECHO "\n===> xz frame tests "
if [ $LZMAMODE -eq 1 ]; then
./datagen > tmp
@@ -520,7 +520,7 @@ else
$ECHO "xz mode not supported"
fi
-$ECHO "\n**** lz4 compatibility tests **** "
+$ECHO "\n===> lz4 compatibility tests "
LZ4MODE=1
$ZSTD --format=lz4 -V || LZ4MODE=0
@@ -543,7 +543,7 @@ else
fi
-$ECHO "\n**** lz4 frame tests **** "
+$ECHO "\n===> lz4 frame tests "
if [ $LZ4MODE -eq 1 ]; then
./datagen > tmp
@@ -556,7 +556,7 @@ else
$ECHO "lz4 mode not supported"
fi
-$ECHO "\n**** zstd round-trip tests **** "
+$ECHO "\n===> zstd round-trip tests "
roundTripTest
roundTripTest -g15K # TableID==3
@@ -569,7 +569,7 @@ roundTripTest -g516K 19 # btopt
fileRoundTripTest -g500K
-$ECHO "\n**** zstd long distance matching round-trip tests **** "
+$ECHO "\n===> zstd long distance matching round-trip tests "
roundTripTest -g0 "2 --long"
roundTripTest -g1000K "1 --long"
roundTripTest -g517K "6 --long"
@@ -580,45 +580,43 @@ fileRoundTripTest -g5M "3 --long"
if [ -n "$hasMT" ]
then
- $ECHO "\n**** zstdmt round-trip tests **** "
+ $ECHO "\n===> zstdmt round-trip tests "
roundTripTest -g4M "1 -T0"
roundTripTest -g8M "3 -T2"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M"
- $ECHO "\n**** zstdmt long distance matching round-trip tests **** "
+ $ECHO "\n===> zstdmt long distance matching round-trip tests "
roundTripTest -g8M "3 --long -T2"
else
- $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+ $ECHO "\n===> no multithreading, skipping zstdmt tests "
fi
rm tmp*
-$ECHO "\n**** zstd --list/-l single frame tests ****"
+$ECHO "\n===> zstd --list/-l single frame tests "
./datagen > tmp1
./datagen > tmp2
./datagen > tmp3
$ZSTD tmp*
$ZSTD -l *.zst
-$ZSTD -lv *.zst
+$ZSTD -lv *.zst | grep "Decompressed Size:" # check that decompressed size is present in header
$ZSTD --list *.zst
$ZSTD --list -v *.zst
-$ECHO "\n**** zstd --list/-l multiple frame tests ****"
+$ECHO "\n===> zstd --list/-l multiple frame tests "
cat tmp1.zst tmp2.zst > tmp12.zst
cat tmp12.zst tmp3.zst > tmp123.zst
$ZSTD -l *.zst
$ZSTD -lv *.zst
-$ZSTD --list *.zst
-$ZSTD --list -v *.zst
-$ECHO "\n**** zstd --list/-l error detection tests ****"
+$ECHO "\n===> zstd --list/-l error detection tests "
! $ZSTD -l tmp1 tmp1.zst
! $ZSTD --list tmp*
! $ZSTD -lv tmp1*
! $ZSTD --list -v tmp2 tmp12.zst
-$ECHO "\n**** zstd --list/-l test with null files ****"
+$ECHO "\n===> zstd --list/-l test with null files "
./datagen -g0 > tmp5
$ZSTD tmp5
$ZSTD -l tmp5.zst
@@ -626,12 +624,12 @@ $ZSTD -l tmp5.zst
$ZSTD -lv tmp5.zst
! $ZSTD -lv tmp5*
-$ECHO "\n**** zstd --list/-l test with no content size field ****"
-./datagen -g1MB | $ZSTD > tmp6.zst
+$ECHO "\n===> zstd --list/-l test with no content size field "
+./datagen -g513K | $ZSTD > tmp6.zst
$ZSTD -l tmp6.zst
-$ZSTD -lv tmp6.zst
+! $ZSTD -lv tmp6.zst | grep "Decompressed Size:" # must NOT be present in header
-$ECHO "\n**** zstd --list/-l test with no checksum ****"
+$ECHO "\n===> zstd --list/-l test with no checksum "
$ZSTD -f --no-check tmp1
$ZSTD -l tmp1.zst
$ZSTD -lv tmp1.zst
@@ -639,7 +637,7 @@ $ZSTD -lv tmp1.zst
rm tmp*
-$ECHO "\n**** zstd long distance matching tests **** "
+$ECHO "\n===> zstd long distance matching tests "
roundTripTest -g0 " --long"
roundTripTest -g9M "2 --long"
# Test parameter parsing
@@ -654,7 +652,7 @@ if [ "$1" != "--test-large-data" ]; then
exit 0
fi
-$ECHO "\n**** large files tests **** "
+$ECHO "\n===> large files tests "
roundTripTest -g270000000 1
roundTripTest -g250000000 2
@@ -685,7 +683,7 @@ roundTripTest -g5000000000 -P99 1
fileRoundTripTest -g4193M -P99 1
-$ECHO "\n**** zstd long, long distance matching round-trip tests **** "
+$ECHO "\n===> zstd long, long distance matching round-trip tests "
roundTripTest -g270000000 "1 --long"
roundTripTest -g130000000 -P60 "5 --long"
roundTripTest -g35000000 -P70 "8 --long"
@@ -697,7 +695,7 @@ roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28"
if [ -n "$hasMT" ]
then
- $ECHO "\n**** zstdmt long round-trip tests **** "
+ $ECHO "\n===> zstdmt long round-trip tests "
roundTripTest -g80000000 -P99 "19 -T2" " "
roundTripTest -g5000000000 -P99 "1 -T2" " "
roundTripTest -g500000000 -P97 "1 -T999" " "
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 1b42bd5d8..6f1f6df18 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -254,9 +254,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
- CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) );
+ CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
outBuff.dst = (char*)(compressedBuffer)+cSize;
- outBuff.size = compressedBufferSize;
+ assert(compressedBufferSize > cSize);
+ outBuff.size = compressedBufferSize - cSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
@@ -622,7 +623,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1;
- CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) );
+ CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
} /* cstream advanced shall write content size = 0 */
inBuff.src = CNBuffer;
inBuff.size = 0;
@@ -935,10 +936,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
- { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+ { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+ params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} }
@@ -1189,13 +1191,13 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
- { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+ { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
params.cParams.windowLog, (U32)pledgedSrcSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
- params.fParams.contentSizeFlag = pledgedSrcSize>0;
+ params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );