ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed Btree update

Browse Source 
ZSTD_updateTree() expected to be followed by a Bt match finder, which would update zc->nextToUpdate.
With the new optimal match finder, it's not necessarily the case : a match might be found during repcode or hash3, and stops there because it reaches sufficient_len, without even entering the binary tree.
Previous policy was to nonetheless update zc->nextToUpdate, but the current position would not be inserted, creating "holes" in the btree, aka positions that will no longer be searched.
Now, when current position is not inserted, zc->nextToUpdate is not update, expecting ZSTD_updateTree() to fill the tree later on.

Solution selected is that ZSTD_updateTree() takes care of properly setting zc->nextToUpdate,
so that it no longer depends on a future function to do this job.

It took time to get there, as the issue started with a memory sanitizer error.
The pb would have been easier to spot with a proper `assert()`.
So this patch add a few of them.

Additionnally, I discovered that `make test` does not enable `assert()` during CLI tests.
This patch enables them.

Unfortunately, these `assert()` triggered other (unrelated) bugs during CLI tests, mostly within zstdmt.
So this patch also fixes them.

- Changed packed structure for gcc memory access : memory sanitizer would complain that a read "might" reach out-of-bound position on the ground that the `union` is larger than the type accessed.
  Now, to avoid this issue, each type is independent.
- ZSTD_CCtxParams_setParameter() : @return provides the value of parameter, clamped/fixed appropriately.
- ZSTDMT : changed constant name to ZSTDMT_JOBSIZE_MIN
- ZSTDMT : multithreading is automatically disabled when srcSize <= ZSTDMT_JOBSIZE_MIN, since only one thread will be used in this case (saves memory and runtime).
- ZSTDMT : nbThreads is automatically clamped on setting the value.
This commit is contained in:
Yann Collet
2017-11-16 12:18:56 -08:00
parent dfc14579f5
commit 05dffe43a7
9 changed files with 263 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+134 -102
View File
@@ -142,17 +142,17 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor
#define ZSTD_CLEVEL_CUSTOM 999
static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
ZSTD_CCtx_params params, U64 srcSizeHint, size_t dictSize)
ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize)
{
return (params.compressionLevel == ZSTD_CLEVEL_CUSTOM ?
params.cParams :
ZSTD_getCParams(params.compressionLevel, srcSizeHint, dictSize));
return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ?
CCtxParams.cParams :
ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize);
}
static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* params, U64 srcSize)
static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize)
{
params->cParams = ZSTD_getCParamsFromCCtxParams(*params, srcSize, 0);
params->compressionLevel = ZSTD_CLEVEL_CUSTOM;
CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0);
CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
}
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
@@ -161,9 +161,9 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1);
}
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params)
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams)
{
ZSTD_cLevelToCCtxParams_srcSize(params, 0);
ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, 0);
}
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
@@ -314,148 +314,157 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
}
size_t ZSTD_CCtxParam_setParameter(
ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value)
ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned value)
{
switch(param)
{
case ZSTD_p_format :
if (value > (unsigned)ZSTD_f_zstd1_magicless)
return ERROR(parameter_unsupported);
params->format = (ZSTD_format_e)value;
return 0;
CCtxParams->format = (ZSTD_format_e)value;
return (size_t)CCtxParams->format;
case ZSTD_p_compressionLevel :
if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();
if (value == 0) return 0;
params->compressionLevel = value;
return 0;
if (value) /* 0 : does not change current level */
CCtxParams->compressionLevel = value;
return CCtxParams->compressionLevel;
case ZSTD_p_windowLog :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
ZSTD_cLevelToCCtxParams(params);
params->cParams.windowLog = value;
return 0;
if (value) { /* 0 : does not change current windowLog */
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.windowLog = value;
}
return CCtxParams->cParams.windowLog;
case ZSTD_p_hashLog :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
ZSTD_cLevelToCCtxParams(params);
params->cParams.hashLog = value;
return 0;
if (value) { /* 0 : does not change current hashLog */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.hashLog = value;
}
return CCtxParams->cParams.hashLog;
case ZSTD_p_chainLog :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
ZSTD_cLevelToCCtxParams(params);
params->cParams.chainLog = value;
return 0;
if (value) { /* 0 : does not change current chainLog */
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.chainLog = value;
}
return CCtxParams->cParams.chainLog;
case ZSTD_p_searchLog :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
ZSTD_cLevelToCCtxParams(params);
params->cParams.searchLog = value;
return 0;
if (value) { /* 0 : does not change current searchLog */
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.searchLog = value;
}
return value;
case ZSTD_p_minMatch :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
ZSTD_cLevelToCCtxParams(params);
params->cParams.searchLength = value;
return 0;
if (value) { /* 0 : does not change current minMatch length */
CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.searchLength = value;
}
return CCtxParams->cParams.searchLength;
case ZSTD_p_targetLength :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
ZSTD_cLevelToCCtxParams(params);
params->cParams.targetLength = value;
return 0;
if (value) { /* 0 : does not change current sufficient_len */
CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.targetLength = value;
}
return CCtxParams->cParams.targetLength;
case ZSTD_p_compressionStrategy :
if (value == 0) return 0;
CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
ZSTD_cLevelToCCtxParams(params);
params->cParams.strategy = (ZSTD_strategy)value;
return 0;
if (value) { /* 0 : does not change currentstrategy */
CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.strategy = (ZSTD_strategy)value;
}
return (size_t)CCtxParams->cParams.strategy;
case ZSTD_p_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */
DEBUGLOG(4, "set content size flag = %u", (value>0));
params->fParams.contentSizeFlag = value > 0;
return 0;
CCtxParams->fParams.contentSizeFlag = value > 0;
return CCtxParams->fParams.contentSizeFlag;
case ZSTD_p_checksumFlag :
/* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
params->fParams.checksumFlag = value > 0;
return 0;
CCtxParams->fParams.checksumFlag = value > 0;
return CCtxParams->fParams.checksumFlag;
case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
params->fParams.noDictIDFlag = (value == 0);
return 0;
CCtxParams->fParams.noDictIDFlag = (value == 0);
return !CCtxParams->fParams.noDictIDFlag;
case ZSTD_p_forceMaxWindow :
params->forceWindow = value > 0;
return 0;
CCtxParams->forceWindow = (value > 0);
return CCtxParams->forceWindow;
case ZSTD_p_nbThreads :
if (value == 0) return 0;
if (value == 0) return CCtxParams->nbThreads;
#ifndef ZSTD_MULTITHREAD
if (value > 1) return ERROR(parameter_unsupported);
return 0;
return 1;
#else
return ZSTDMT_initializeCCtxParameters(params, value);
return ZSTDMT_CCtxParam_setNbThreads(CCtxParams, value);
#endif
case ZSTD_p_jobSize :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
if (params->nbThreads <= 1) return ERROR(parameter_unsupported);
return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_sectionSize, value);
if (CCtxParams->nbThreads <= 1) return ERROR(parameter_unsupported);
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_sectionSize, value);
#endif
case ZSTD_p_overlapSizeLog :
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
if (params->nbThreads <= 1) return ERROR(parameter_unsupported);
return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value);
if (CCtxParams->nbThreads <= 1) return ERROR(parameter_unsupported);
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value);
#endif
case ZSTD_p_enableLongDistanceMatching :
if (value != 0) {
ZSTD_cLevelToCCtxParams(params);
params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
}
return ZSTD_ldm_initializeParameters(&params->ldmParams, value);
return ZSTD_ldm_initializeParameters(&CCtxParams->ldmParams, value);
case ZSTD_p_ldmHashLog :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
params->ldmParams.hashLog = value;
return 0;
if (value) { /* 0 : does not change current ldmHashLog */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CCtxParams->ldmParams.hashLog = value;
}
return CCtxParams->ldmParams.hashLog;
case ZSTD_p_ldmMinMatch :
if (value == 0) return 0;
CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
params->ldmParams.minMatchLength = value;
return 0;
if (value) { /* 0 : does not change current ldmMinMatch */
CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
CCtxParams->ldmParams.minMatchLength = value;
}
return CCtxParams->ldmParams.minMatchLength;
case ZSTD_p_ldmBucketSizeLog :
if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) {
return ERROR(parameter_outOfBound);
}
params->ldmParams.bucketSizeLog = value;
return 0;
CCtxParams->ldmParams.bucketSizeLog = value;
return value;
case ZSTD_p_ldmHashEveryLog :
if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) {
return ERROR(parameter_outOfBound);
}
params->ldmParams.hashEveryLog = value;
return 0;
CCtxParams->ldmParams.hashEveryLog = value;
return value;
default: return ERROR(parameter_unsupported);
}
@@ -768,12 +777,26 @@ static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1,
ldmParams1.hashEveryLog == ldmParams2.hashEveryLog);
}
typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
/* ZSTD_equivalentBuffPolicy() :
* check internal buffers exist for streaming if buffPol == ZSTDb_buffered .
* Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */
static U32 ZSTD_equivalentBuffPolicy(size_t bufferSize, ZSTD_buffered_policy_e buffPol)
{
if ((buffPol == ZSTDb_buffered) & (!bufferSize)) return 0;
return 1;
}
/** Equivalence for resetCCtx purposes */
static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1,
ZSTD_CCtx_params params2)
ZSTD_CCtx_params params2,
size_t buffSize1,
ZSTD_buffered_policy_e buffPol2)
{
return ZSTD_equivalentCParams(params1.cParams, params2.cParams) &&
ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams);
ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams) &&
ZSTD_equivalentBuffPolicy(buffSize1, buffPol2);
}
/*! ZSTD_continueCCtx() :
@@ -802,7 +825,6 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl
}
typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
/*! ZSTD_resetCCtx_internal() :
note : `params` are assumed fully validated at this stage */
@@ -815,8 +837,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
if (crp == ZSTDcrp_continue) {
if (ZSTD_equivalentParams(params, zc->appliedParams)) {
DEBUGLOG(4, "ZSTD_equivalentParams()==1");
if (ZSTD_equivalentParams(params, zc->appliedParams, zc->inBuffSize, zbuff)) {
DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode");
assert(!(params.ldmParams.enableLdm &&
params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET));
zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
@@ -825,6 +847,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->entropy->litlength_repeatMode = FSE_repeat_none;
return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
} }
DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
if (params.ldmParams.enableLdm) {
/* Adjust long distance matching parameters */
@@ -863,11 +886,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
: 0;
size_t const neededSpace = entropySpace + optSpace + ldmSpace +
tableSpace + tokenSpace + bufferSpace;
DEBUGLOG(4, "Need %uKB workspace, including %uKB for tables",
(U32)(neededSpace>>10), (U32)(tableSpace>>10));
DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
(U32)chainSize, (U32)hSize, (U32)h3Size);
if (zc->workSpaceSize < neededSpace) { /* too small : resize */
DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n",
(unsigned)zc->workSpaceSize>>10,
(unsigned)neededSpace>>10);
DEBUGLOG(4, "Need to update workSpaceSize from %uK to %uK",
(unsigned)(zc->workSpaceSize>>10),
(unsigned)(neededSpace>>10));
/* static cctx : no resize, error out */
if (zc->staticSize) return ERROR(memory_allocation);
@@ -916,7 +943,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
/* opt parser space */
if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
DEBUGLOG(5, "reserving optimal parser space");
DEBUGLOG(4, "reserving optimal parser space");
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->optState.litFreq = (U32*)ptr;
zc->optState.litLengthFreq = zc->optState.litFreq + (1<<Litbits);
@@ -940,6 +967,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
}
/* table Space */
DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset);
if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->hashTable = (U32*)(ptr);
@@ -1753,7 +1781,6 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
!params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
if (params.format == ZSTD_f_zstd1) {
DEBUGLOG(4, "writing zstd magic number");
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
pos = 4;
}
@@ -1787,8 +1814,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
const BYTE* const ip = (const BYTE*) src;
size_t fhSize = 0;
DEBUGLOG(5, "ZSTD_compressContinue_internal");
DEBUGLOG(5, "stage: %u", cctx->stage);
DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u", cctx->stage);
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
if (frame && (cctx->stage==ZSTDcs_init)) {
@@ -2017,7 +2043,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode)
{
DEBUGLOG(5, "ZSTD_compress_insertDictionary");
DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
if ((dict==NULL) || (dictSize<=8)) return 0;
/* dict restricted modes */
@@ -2026,7 +2052,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
if (dictMode == ZSTD_dm_auto) {
DEBUGLOG(5, "raw content dictionary detected");
DEBUGLOG(4, "raw content dictionary detected");
return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
}
if (dictMode == ZSTD_dm_fullDict)
@@ -2097,6 +2123,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
DEBUGLOG(4, "ZSTD_compressBegin_usingDict");
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
}
@@ -2179,6 +2206,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
{
ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
DEBUGLOG(4, "ZSTD_compress_internal");
return ZSTD_compress_advanced_internal(cctx,
dst, dstCapacity,
src, srcSize,
@@ -2192,6 +2220,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
const void* dict,size_t dictSize,
ZSTD_parameters params)
{
DEBUGLOG(4, "ZSTD_compress_advanced");
CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
}
@@ -2204,6 +2233,7 @@ size_t ZSTD_compress_advanced_internal(
const void* dict,size_t dictSize,
ZSTD_CCtx_params params)
{
DEBUGLOG(4, "ZSTD_compress_advanced_internal");
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
params, srcSize, ZSTDb_not_buffered) );
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
@@ -2214,6 +2244,8 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co
{
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize ? srcSize : 1, dict ? dictSize : 0);
params.fParams.contentSizeFlag = 1;
DEBUGLOG(4, "ZSTD_compress_usingDict (level=%i, srcSize=%u, dictSize=%u)",
compressionLevel, (U32)srcSize, (U32)dictSize);
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
}
@@ -2270,7 +2302,7 @@ static size_t ZSTD_initCDict_internal(
ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams)
{
DEBUGLOG(5, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
DEBUGLOG(4, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer;
@@ -2409,11 +2441,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
{
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
if (cdict==NULL) return ERROR(dictionary_wrong);
{ ZSTD_CCtx_params params = cctx->requestedParams;
params.cParams = ZSTD_getCParamsFromCDict(cdict);
params.fParams = fParams;
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto,
cdict,
@@ -2811,16 +2843,17 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
/* transparent initialization stage */
if (cctx->streamStage == zcss_init) {
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
ZSTD_CCtx_params params = cctx->requestedParams;
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1;
params.cParams = ZSTD_getCParamsFromCCtxParams(
cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage");
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */
params.cParams = ZSTD_getCParamsFromCCtxParams(
cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
#ifdef ZSTD_MULTITHREAD
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) params.nbThreads = 1; /* do not invoke multi-threading when src size is too small */
if (params.nbThreads > 1) {
if (cctx->mtctx == NULL || cctx->appliedParams.nbThreads != params.nbThreads) {
ZSTDMT_freeCCtx(cctx->mtctx);
@@ -2897,8 +2930,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4;
size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize;
DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u",
(unsigned)toFlush);
DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (U32)toFlush);
return toFlush;
}
}