ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move dictMode to cctxParams

Browse Source
This commit is contained in:
Stella Lau
2017-08-18 11:43:31 -07:00
parent 97e27affcb
commit b6cb2ed8cb
3 changed files with 47 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/zstd_internal.h
+2
View File
@@ -225,6 +225,8 @@ typedef struct ZSTD_CCtx_params_s {
ZSTD_compressionParameters cParams; ZSTD_compressionParameters cParams;
ZSTD_frameParameters fParams; ZSTD_frameParameters fParams;
int compressionLevel; int compressionLevel;
U32 forceWindow;
ZSTD_dictMode_e dictMode;
} ZSTD_CCtx_params; } ZSTD_CCtx_params;
lib/compress/zstd_compress.c
+44 -32
View File
@@ -80,7 +80,7 @@ struct ZSTD_CCtx_s {
U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */ U32 loadedDictEnd; /* index of end of dictionary */
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ // U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
ZSTD_compressionStage_e stage; ZSTD_compressionStage_e stage;
U32 dictID; U32 dictID;
// int compressionLevel; // int compressionLevel;
@@ -116,7 +116,7 @@ struct ZSTD_CCtx_s {
U32 frameEnded; U32 frameEnded;
/* Dictionary */ /* Dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */ // ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
U32 dictContentByRef; U32 dictContentByRef;
ZSTD_CDict* cdictLocal; ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict; const ZSTD_CDict* cdict;
@@ -240,10 +240,15 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
{ {
switch(param) switch(param)
{ {
case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0; case ZSTD_p_forceWindow :
cctx->requestedParams.forceWindow = value>0;
cctx->loadedDictEnd = 0;
return 0;
ZSTD_STATIC_ASSERT(ZSTD_dm_auto==0); ZSTD_STATIC_ASSERT(ZSTD_dm_auto==0);
ZSTD_STATIC_ASSERT(ZSTD_dm_rawContent==1); ZSTD_STATIC_ASSERT(ZSTD_dm_rawContent==1);
case ZSTD_p_forceRawDict : cctx->dictMode = (ZSTD_dictMode_e)(value>0); return 0; case ZSTD_p_forceRawDict :
cctx->requestedParams.dictMode = (ZSTD_dictMode_e)(value>0);
return 0;
default: return ERROR(parameter_unsupported); default: return ERROR(parameter_unsupported);
} }
} }
@@ -407,7 +412,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent); ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
if (value > (unsigned)ZSTD_dm_fullDict) if (value > (unsigned)ZSTD_dm_fullDict)
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
cctx->dictMode = (ZSTD_dictMode_e)value; cctx->requestedParams.dictMode = (ZSTD_dictMode_e)value;
return 0; return 0;
case ZSTD_p_refDictContent : case ZSTD_p_refDictContent :
@@ -419,7 +424,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content * even when referencing into Dictionary content
* default : 0 when using a CDict, 1 when using a Prefix */ * default : 0 when using a CDict, 1 when using a Prefix */
cctx->forceWindow = value>0; cctx->requestedParams.forceWindow = value>0;
cctx->loadedDictEnd = 0; cctx->loadedDictEnd = 0;
return 0; return 0;
@@ -532,7 +537,7 @@ size_t ZSTD_CCtxParam_setParameter(
if (value > (unsigned)ZSTD_dm_fullDict) { if (value > (unsigned)ZSTD_dm_fullDict) {
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
} }
// cctx->dictMode = (ZSTD_dictMode_e)value; params->dictMode = (ZSTD_dictMode_e)value;
return 0; return 0;
case ZSTD_p_refDictContent : case ZSTD_p_refDictContent :
@@ -540,7 +545,7 @@ size_t ZSTD_CCtxParam_setParameter(
return 0; return 0;
case ZSTD_p_forceMaxWindow : case ZSTD_p_forceMaxWindow :
// cctx->forceWindow = value > 0; params->forceWindow = value > 0;
return 0; return 0;
case ZSTD_p_nbThreads : case ZSTD_p_nbThreads :
@@ -587,7 +592,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
ZSTD_getCParams(cctx->requestedParams.compressionLevel, 0, dictSize); ZSTD_getCParams(cctx->requestedParams.compressionLevel, 0, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced( cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize, dict, dictSize,
cctx->dictContentByRef, cctx->dictMode, cctx->dictContentByRef, cctx->requestedParams.dictMode,
cParams, cctx->customMem); cParams, cctx->customMem);
cctx->cdict = cctx->cdictLocal; cctx->cdict = cctx->cdictLocal;
if (cctx->cdictLocal == NULL) if (cctx->cdictLocal == NULL)
@@ -3244,7 +3249,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->dictBase = zc->base; zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc; zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit; zc->nextToUpdate = zc->dictLimit;
zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base); zc->loadedDictEnd = zc->appliedParams.forceWindow ? 0 : (U32)(iend - zc->base);
zc->nextSrc = iend; zc->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0; if (srcSize <= HASH_READ_SIZE) return 0;
@@ -3417,14 +3422,13 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_CCtx_params params, U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff) ZSTD_buffered_policy_e zbuff)
{ {
DEBUGLOG(4, "ZSTD_compressBegin_internal"); DEBUGLOG(4, "ZSTD_compressBegin_internal");
DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none")); DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none"));
DEBUGLOG(4, "dictMode : %u", (U32)dictMode); DEBUGLOG(4, "dictMode : %u", (U32)(params.dictMode));
/* params are supposed to be fully validated at this point */ /* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@@ -3437,7 +3441,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
ZSTDcrp_continue, zbuff) ); ZSTDcrp_continue, zbuff) );
return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode); return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode);
} }
@@ -3451,9 +3455,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
ZSTD_CCtx_params cctxParams = cctx->requestedParams; ZSTD_CCtx_params cctxParams = cctx->requestedParams;
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
cctxParams.dictMode = ZSTD_dm_auto;
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F(ZSTD_checkCParams(params.cParams)); CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
cctxParams, pledgedSrcSize, ZSTDb_not_buffered); cctxParams, pledgedSrcSize, ZSTDb_not_buffered);
} }
@@ -3544,8 +3549,9 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
ZSTD_CCtx_params cctxParams = cctx->requestedParams; ZSTD_CCtx_params cctxParams = cctx->requestedParams;
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
cctxParams.dictMode = ZSTD_dm_auto;
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
cctxParams, srcSize, ZSTDb_not_buffered) ); cctxParams, srcSize, ZSTDb_not_buffered) );
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
@@ -3621,6 +3627,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
} }
#if 0
static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
{ {
ZSTD_parameters params; ZSTD_parameters params;
@@ -3628,6 +3635,7 @@ static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_
params.fParams = fParams; params.fParams = fParams;
return params; return params;
} }
#endif
static size_t ZSTD_initCDict_internal( static size_t ZSTD_initCDict_internal(
ZSTD_CDict* cdict, ZSTD_CDict* cdict,
@@ -3650,13 +3658,12 @@ static size_t ZSTD_initCDict_internal(
{ ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */
ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams); ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams;
ZSTD_CCtx_params cctxParams = cctxParams.cParams = cParams;
ZSTD_makeCCtxParamsFromParams(params); cctxParams.fParams = fParams;
cctxParams.compressionLevel = cctxParams.dictMode = dictMode;
cdict->refContext->requestedParams.compressionLevel;
CHECK_F( ZSTD_compressBegin_internal(cdict->refContext, CHECK_F( ZSTD_compressBegin_internal(cdict->refContext,
cdict->dictContent, dictSize, dictMode, cdict->dictContent, dictSize,
NULL, NULL,
cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN,
ZSTDb_not_buffered) ); ZSTDb_not_buffered) );
@@ -3723,7 +3730,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque( ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque(
void *workspace, size_t workspaceSize, const void* dict, void *workspace, size_t workspaceSize, const void* dict,
size_t dictSize, unsigned byReference, ZSTD_dictMode_e dictMode, size_t dictSize, unsigned byReference,
ZSTD_CCtx_params* params) ZSTD_CCtx_params* params)
{ {
ZSTD_compressionParameters cParams = params->cParams; ZSTD_compressionParameters cParams = params->cParams;
@@ -3749,7 +3756,7 @@ ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque(
if (ZSTD_isError( ZSTD_initCDict_internal(cdict, if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize, dict, dictSize,
1 /* byReference */, dictMode, 1 /* byReference */, params->dictMode,
cParams) )) cParams) ))
return NULL; return NULL;
@@ -3775,9 +3782,10 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
ZSTD_CCtx_params params = ZSTD_makeCCtxParamsFromCParams(cParams); ZSTD_CCtx_params params = ZSTD_makeCCtxParamsFromCParams(cParams);
params.dictMode = dictMode;
return ZSTD_initStaticCDict_advanced_opaque( return ZSTD_initStaticCDict_advanced_opaque(
workspace, workspaceSize, dict, dictSize, workspace, workspaceSize, dict, dictSize,
byReference, dictMode, &params); byReference, &params);
} }
@@ -3799,9 +3807,10 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
{ {
ZSTD_CCtx_params params = cdict->refContext->appliedParams; ZSTD_CCtx_params params = cdict->refContext->appliedParams;
params.fParams = fParams; params.fParams = fParams;
params.dictMode = ZSTD_dm_auto;
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
return ZSTD_compressBegin_internal(cctx, return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto, NULL, 0,
cdict, cdict,
params, pledgedSrcSize, params, pledgedSrcSize,
ZSTDb_not_buffered); ZSTDb_not_buffered);
@@ -3885,13 +3894,14 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
{ {
ZSTD_CCtx_params cctxParams = ZSTD_makeCCtxParamsFromParams(params); ZSTD_CCtx_params cctxParams = ZSTD_makeCCtxParamsFromParams(params);
cctxParams.compressionLevel = zcs->requestedParams.compressionLevel; cctxParams.compressionLevel = zcs->requestedParams.compressionLevel;
cctxParams.dictMode = dictMode;
DEBUGLOG(4, "ZSTD_resetCStream_internal"); DEBUGLOG(4, "ZSTD_resetCStream_internal");
/* params are supposed to be fully validated at this point */ /* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
CHECK_F( ZSTD_compressBegin_internal(zcs, CHECK_F( ZSTD_compressBegin_internal(zcs,
dict, dictSize, dictMode, dict, dictSize,
cdict, cdict,
cctxParams, pledgedSrcSize, cctxParams, pledgedSrcSize,
ZSTDb_buffered) ); ZSTDb_buffered) );
@@ -3913,7 +3923,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
if (zcs->requestedParams.compressionLevel != ZSTD_CLEVEL_CUSTOM) { if (zcs->requestedParams.compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(zcs->requestedParams.compressionLevel, pledgedSrcSize, 0 /* dictSize */); params.cParams = ZSTD_getCParams(zcs->requestedParams.compressionLevel, pledgedSrcSize, 0 /* dictSize */);
} }
return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->requestedParams.dictMode, zcs->cdict, params, pledgedSrcSize);
} }
/*! ZSTD_initCStream_internal() : /*! ZSTD_initCStream_internal() :
@@ -3936,7 +3946,8 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
} }
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
zcs->dictContentByRef, zcs->dictMode, zcs->dictContentByRef,
zcs->requestedParams.dictMode,
params.cParams, zcs->customMem); params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@@ -3949,10 +3960,11 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
zcs->cdictLocal = NULL; zcs->cdictLocal = NULL;
zcs->cdict = cdict; zcs->cdict = cdict;
} }
zcs->requestedParams.cParams = params.cParams;
zcs->requestedParams = ZSTD_makeCCtxParamsFromParams(params); zcs->requestedParams.fParams = params.fParams;
zcs->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; zcs->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->requestedParams.dictMode, zcs->cdict, params, pledgedSrcSize);
} }
/* ZSTD_initCStream_usingCDict_advanced() : /* ZSTD_initCStream_usingCDict_advanced() :
@@ -4212,7 +4224,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
} else } else
#endif #endif
{ {
CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->requestedParams.dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
} } } }
/* compression stage */ /* compression stage */
lib/zstd.h
+1 -1
View File
@@ -608,7 +608,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque( ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode, unsigned byReference,
ZSTD_CCtx_params* params); ZSTD_CCtx_params* params);
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);