ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2311 from felixhandte/ddss-fix-cparam-derivation

Browse Source 
Fix Compression Parameter Derivation Bugs Introduced by DDSS Changes
This commit is contained in:
Felix Handte
2020-09-18 14:02:14 -04:00
committed by GitHub
parent 5870c47bc0 f23a321781
commit 200c960f1d
3 changed files with 124 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+79 -75
View File
@@ -192,11 +192,10 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
ZSTD_compressionParameters cParams)
{
ZSTD_CCtx_params cctxParams;
ZSTD_memset(&cctxParams, 0, sizeof(cctxParams));
/* should not matter, as all cParams are presumed properly defined */
ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT);
cctxParams.cParams = cParams;
cctxParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */
assert(!ZSTD_checkCParams(cParams));
cctxParams.fParams.contentSizeFlag = 1;
return cctxParams;
}
@@ -208,9 +207,8 @@ static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced(
params = (ZSTD_CCtx_params*)ZSTD_customCalloc(
sizeof(ZSTD_CCtx_params), customMem);
if (!params) { return NULL; }
ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT);
params->customMem = customMem;
params->compressionLevel = ZSTD_CLEVEL_DEFAULT;
params->fParams.contentSizeFlag = 1;
return params;
}
@@ -869,9 +867,7 @@ static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(
unsigned long long srcSizeHint,
size_t const dictSize);
static int ZSTD_dedicatedDictSearch_isSupported(
int const compressionLevel,
unsigned long long srcSizeHint,
size_t const dictSize);
const ZSTD_compressionParameters* cParams);
static void ZSTD_dedicatedDictSearch_revertCParams(
ZSTD_compressionParameters* cParams);
@@ -1098,6 +1094,19 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize);
static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize);
static void ZSTD_overrideCParams(
ZSTD_compressionParameters* cParams,
const ZSTD_compressionParameters* overrides)
{
if (overrides->windowLog) cParams->windowLog = overrides->windowLog;
if (overrides->hashLog) cParams->hashLog = overrides->hashLog;
if (overrides->chainLog) cParams->chainLog = overrides->chainLog;
if (overrides->searchLog) cParams->searchLog = overrides->searchLog;
if (overrides->minMatch) cParams->minMatch = overrides->minMatch;
if (overrides->targetLength) cParams->targetLength = overrides->targetLength;
if (overrides->strategy) cParams->strategy = overrides->strategy;
}
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize)
{
@@ -1107,13 +1116,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
}
cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize);
if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog;
if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog;
if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog;
if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog;
if (CCtxParams->cParams.minMatch) cParams.minMatch = CCtxParams->cParams.minMatch;
if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength;
if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy;
ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
assert(!ZSTD_checkCParams(cParams));
/* srcSizeHint == 0 means 0 */
return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize);
@@ -3411,12 +3414,11 @@ static size_t ZSTD_initCDict_internal(
const void* dictBuffer, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams,
ZSTD_CCtx_params params)
{
DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType);
assert(!ZSTD_checkCParams(cParams));
cdict->matchState.cParams = cParams;
assert(!ZSTD_checkCParams(params.cParams));
cdict->matchState.cParams = params.cParams;
cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch;
if (cdict->matchState.dedicatedDictSearch && dictSize > ZSTD_CHUNKSIZE_MAX) {
cdict->matchState.dedicatedDictSearch = 0;
@@ -3439,7 +3441,7 @@ static size_t ZSTD_initCDict_internal(
FORWARD_IF_ERROR(ZSTD_reset_matchState(
&cdict->matchState,
&cdict->workspace,
&cParams,
&params.cParams,
ZSTDcrp_makeClean,
ZSTDirp_reset,
ZSTD_resetTarget_CDict), "");
@@ -3448,7 +3450,6 @@ static size_t ZSTD_initCDict_internal(
*/
{ params.compressionLevel = ZSTD_CLEVEL_DEFAULT;
params.fParams.contentSizeFlag = 1;
params.cParams = cParams;
{ size_t const dictID = ZSTD_compress_insertDictionary(
&cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
&params, cdict->dictContent, cdict->dictContentSize,
@@ -3498,63 +3499,65 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
ZSTD_compressionParameters cParams,
ZSTD_customMem customMem)
{
DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* cdict = ZSTD_createCDict_advanced_internal(dictSize,
dictLoadMethod, cParams,
customMem);
ZSTD_CCtx_params params;
ZSTD_memset(&params, 0, sizeof(params));
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dictBuffer, dictSize,
dictLoadMethod, dictContentType,
cParams, params) )) {
ZSTD_freeCDict(cdict);
return NULL;
}
return cdict;
}
ZSTD_CCtx_params cctxParams;
memset(&cctxParams, 0, sizeof(cctxParams));
ZSTD_CCtxParams_init(&cctxParams, 0);
cctxParams.cParams = cParams;
cctxParams.customMem = customMem;
return ZSTD_createCDict_advanced2(
dictBuffer, dictSize,
dictLoadMethod, dictContentType,
&cctxParams, customMem);
}
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_CCtx_params* cctxParams,
ZSTD_customMem customMem)
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
const ZSTD_CCtx_params* originalCctxParams,
ZSTD_customMem customMem)
{
int const dedicatedDictSearch = cctxParams->enableDedicatedDictSearch &&
ZSTD_dedicatedDictSearch_isSupported(
cctxParams->compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
if (!dedicatedDictSearch) {
ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(
cctxParams->compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
return ZSTD_createCDict_advanced(dict, dictSize,
dictLoadMethod, dictContentType, cParams,
customMem);
}
{ ZSTD_compressionParameters const cParams = ZSTD_dedicatedDictSearch_getCParams(
cctxParams->compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
ZSTD_CDict* cdict = ZSTD_createCDict_advanced_internal(dictSize,
dictLoadMethod, cParams,
customMem);
ZSTD_CCtx_params cctxParams = *originalCctxParams;
ZSTD_compressionParameters cParams;
ZSTD_CDict* cdict;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize,
dictLoadMethod, dictContentType,
cParams, *cctxParams) )) {
ZSTD_freeCDict(cdict);
return NULL;
}
DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
return cdict;
if (cctxParams.enableDedicatedDictSearch) {
cParams = ZSTD_dedicatedDictSearch_getCParams(
cctxParams.compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
ZSTD_overrideCParams(&cParams, &cctxParams.cParams);
} else {
cParams = ZSTD_getCParamsFromCCtxParams(
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
}
if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) {
/* Fall back to non-DDSS params */
cctxParams.enableDedicatedDictSearch = 0;
cParams = ZSTD_getCParamsFromCCtxParams(
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
}
cctxParams.cParams = cParams;
cdict = ZSTD_createCDict_advanced_internal(dictSize,
dictLoadMethod, cctxParams.cParams,
customMem);
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize,
dictLoadMethod, dictContentType,
cctxParams) )) {
ZSTD_freeCDict(cdict);
return NULL;
}
return cdict;
}
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
@@ -3632,12 +3635,13 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
(unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize));
if (workspaceSize < neededSize) return NULL;
ZSTD_memset(&params, 0, sizeof(params));
ZSTD_CCtxParams_init(&params, 0);
params.cParams = cParams;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize,
dictLoadMethod, dictContentType,
cParams, params) ))
params) ))
return NULL;
return cdict;
@@ -4362,10 +4366,10 @@ static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const
return cParams;
}
static int ZSTD_dedicatedDictSearch_isSupported(int const compressionLevel, unsigned long long srcSizeHint, size_t const dictSize)
static int ZSTD_dedicatedDictSearch_isSupported(
ZSTD_compressionParameters const* cParams)
{
ZSTD_compressionParameters const cParams = ZSTD_dedicatedDictSearch_getCParams(compressionLevel, srcSizeHint, dictSize);
return (cParams.strategy >= ZSTD_greedy) && (cParams.strategy <= ZSTD_lazy2);
return (cParams->strategy >= ZSTD_greedy) && (cParams->strategy <= ZSTD_lazy2);
}
/**
lib/zstd.h
+1 -1
View File
@@ -1412,7 +1412,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_CCtx_params* cctxParams,
const ZSTD_CCtx_params* cctxParams,
ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,