ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

continued: changed to overlapLog

Browse Source 
in deeper code layer.
for consistency.
This commit is contained in:
Yann Collet
2018-12-11 17:41:42 -08:00
parent 9b784dec7f
commit eee789b7ea
5 changed files with 79 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+3 -3
View File
@@ -591,7 +591,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported); return ERROR(parameter_unsupported);
#else #else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value); return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value);
#endif #endif
case ZSTD_c_rsyncable : case ZSTD_c_rsyncable :
@@ -702,7 +702,7 @@ size_t ZSTD_CCtxParam_getParameter(
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported); return ERROR(parameter_unsupported);
#else #else
*value = CCtxParams->overlapSizeLog; *value = CCtxParams->overlapLog;
break; break;
#endif #endif
case ZSTD_c_rsyncable : case ZSTD_c_rsyncable :
@@ -873,7 +873,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams)
CLAMP(ZSTD_c_searchLog, cParams.searchLog); CLAMP(ZSTD_c_searchLog, cParams.searchLog);
CLAMP(ZSTD_c_minMatch, cParams.minMatch); CLAMP(ZSTD_c_minMatch, cParams.minMatch);
CLAMP(ZSTD_c_targetLength,cParams.targetLength); CLAMP(ZSTD_c_targetLength,cParams.targetLength);
CLAMP_TYPE(ZSTD_c_strategy, cParams.strategy, ZSTD_strategy); CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy);
return cParams; return cParams;
} }
lib/compress/zstd_compress_internal.h
+4 -4
View File
@@ -190,10 +190,10 @@ struct ZSTD_CCtx_params_s {
ZSTD_dictAttachPref_e attachDictPref; ZSTD_dictAttachPref_e attachDictPref;
/* Multithreading: used to pass parameters to mtctx */ /* Multithreading: used to pass parameters to mtctx */
unsigned nbWorkers; int nbWorkers;
unsigned jobSize; size_t jobSize;
unsigned overlapSizeLog; int overlapLog;
unsigned rsyncable; int rsyncable;
/* Long distance matching parameters */ /* Long distance matching parameters */
ldmParams_t ldmParams; ldmParams_t ldmParams;
lib/compress/zstdmt_compress.c
+61 -31
View File
@@ -10,7 +10,7 @@
/* ====== Tuning parameters ====== */ /* ====== Tuning parameters ====== */
#define ZSTDMT_OVERLAPLOG_DEFAULT 6 #define ZSTDMT_OVERLAPLOG_DEFAULT 0
/* ====== Compiler specifics ====== */ /* ====== Compiler specifics ====== */
@@ -340,8 +340,8 @@ static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers)
typedef struct { typedef struct {
ZSTD_pthread_mutex_t poolMutex; ZSTD_pthread_mutex_t poolMutex;
unsigned totalCCtx; int totalCCtx;
unsigned availCCtx; int availCCtx;
ZSTD_customMem cMem; ZSTD_customMem cMem;
ZSTD_CCtx* cctx[1]; /* variable size */ ZSTD_CCtx* cctx[1]; /* variable size */
} ZSTDMT_CCtxPool; } ZSTDMT_CCtxPool;
@@ -349,16 +349,16 @@ typedef struct {
/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ /* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
{ {
unsigned u; int cid;
for (u=0; u<pool->totalCCtx; u++) for (cid=0; cid<pool->totalCCtx; cid++)
ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */ ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */
ZSTD_pthread_mutex_destroy(&pool->poolMutex); ZSTD_pthread_mutex_destroy(&pool->poolMutex);
ZSTD_free(pool, pool->cMem); ZSTD_free(pool, pool->cMem);
} }
/* ZSTDMT_createCCtxPool() : /* ZSTDMT_createCCtxPool() :
* implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers, static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
ZSTD_customMem cMem) ZSTD_customMem cMem)
{ {
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
@@ -379,7 +379,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
} }
static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool,
unsigned nbWorkers) int nbWorkers)
{ {
if (srcPool==NULL) return NULL; if (srcPool==NULL) return NULL;
if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */
@@ -866,7 +866,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
{ {
if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX; if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX;
params->nbWorkers = nbWorkers; params->nbWorkers = nbWorkers;
params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT; params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
params->jobSize = 0; params->jobSize = 0;
return nbWorkers; return nbWorkers;
} }
@@ -976,28 +976,36 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
} }
/* Internal only */ /* Internal only */
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, size_t
ZSTDMT_parameter parameter, unsigned value) { ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
ZSTDMT_parameter parameter,
int value)
{
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter"); DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
switch(parameter) switch(parameter)
{ {
case ZSTDMT_p_jobSize : case ZSTDMT_p_jobSize :
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value); DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
if ( (value > 0) /* value==0 => automatic job size */ if (value < ZSTDMT_JOBSIZE_MIN) value = ZSTDMT_JOBSIZE_MIN;
& (value < ZSTDMT_JOBSIZE_MIN) ) assert(value >= 0);
value = ZSTDMT_JOBSIZE_MIN; { size_t jobSize = value;
if (value > ZSTDMT_JOBSIZE_MAX) if (jobSize > ZSTDMT_JOBSIZE_MAX) jobSize = ZSTDMT_JOBSIZE_MAX;
value = ZSTDMT_JOBSIZE_MAX; params->jobSize = jobSize;
params->jobSize = value; return jobSize;
return value; }
case ZSTDMT_p_overlapSectionLog :
if (value > 9) value = 9; case ZSTDMT_p_overlapLog :
DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value); DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %i", value);
params->overlapSizeLog = (value >= 9) ? 9 : value; if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN;
if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX;
params->overlapLog = value;
return value; return value;
case ZSTDMT_p_rsyncable : case ZSTDMT_p_rsyncable :
params->rsyncable = (value == 0 ? 0 : 1); value = (value != 0);
params->rsyncable = value;
return value; return value;
default : default :
return ERROR(parameter_unsupported); return ERROR(parameter_unsupported);
} }
@@ -1015,8 +1023,8 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter,
case ZSTDMT_p_jobSize: case ZSTDMT_p_jobSize:
*value = mtctx->params.jobSize; *value = mtctx->params.jobSize;
break; break;
case ZSTDMT_p_overlapSectionLog: case ZSTDMT_p_overlapLog:
*value = mtctx->params.overlapSizeLog; *value = mtctx->params.overlapLog;
break; break;
case ZSTDMT_p_rsyncable: case ZSTDMT_p_rsyncable:
*value = mtctx->params.rsyncable; *value = mtctx->params.rsyncable;
@@ -1154,7 +1162,7 @@ static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params) static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params)
{ {
unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog; unsigned const overlapRLog = (params.overlapLog>9) ? 0 : 9-params.overlapLog;
if (params.ldmParams.enableLdm) if (params.ldmParams.enableLdm)
return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog); return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog);
return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog); return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog);
@@ -1303,21 +1311,43 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
ZSTD_CCtx_params cctxParams = mtctx->params; ZSTD_CCtx_params cctxParams = mtctx->params;
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
cctxParams.overlapSizeLog = overlapLog; cctxParams.overlapLog = overlapLog;
return ZSTDMT_compress_advanced_internal(mtctx, return ZSTDMT_compress_advanced_internal(mtctx,
dst, dstCapacity, dst, dstCapacity,
src, srcSize, src, srcSize,
cdict, cctxParams); cdict, cctxParams);
} }
static int ZSTDMT_overlapLog_default(ZSTD_strategy strat)
{
switch(strat)
{
case ZSTD_btultra2:
return 9;
case ZSTD_btultra:
case ZSTD_btopt:
return 8;
case ZSTD_btlazy2:
case ZSTD_lazy2:
return 7;
case ZSTD_lazy:
case ZSTD_greedy:
case ZSTD_dfast:
case ZSTD_fast:
default:
return 6;
}
assert(0);
}
size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
int compressionLevel) int compressionLevel)
{ {
U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT;
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog); return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog);
} }
@@ -1381,7 +1411,7 @@ size_t ZSTDMT_initCStream_internal(
} }
mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params); mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(mtctx->targetPrefixSize>>10)); DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetSectionSize = params.jobSize; mtctx->targetSectionSize = params.jobSize;
if (mtctx->targetSectionSize == 0) { if (mtctx->targetSectionSize == 0) {
mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params);
@@ -1397,7 +1427,7 @@ size_t ZSTDMT_initCStream_internal(
mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH);
} }
if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), params.jobSize); DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize);
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10));
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize));
{ {
lib/compress/zstdmt_compress.h
+10 -8
View File
@@ -29,11 +29,13 @@
/* === Constants === */ /* === Constants === */
#define ZSTDMT_NBWORKERS_MAX 200 #ifndef ZSTDMT_NBWORKERS_MAX
#ifndef ZSTDMT_JOBSIZE_MIN # define ZSTDMT_NBWORKERS_MAX 200
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif #endif
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1 GB)) /* note : limited by `jobSize` type, which is `int` */ #ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#endif
#define ZSTDMT_JOBSIZE_MAX ((size_t)(MEM_32bits() ? (512 MB) : (1 GB)))
/* === Memory management === */ /* === Memory management === */
@@ -89,9 +91,9 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
/* ZSTDMT_parameter : /* ZSTDMT_parameter :
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
typedef enum { typedef enum {
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
ZSTDMT_p_overlapSectionLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ ZSTDMT_p_rsyncable /* Enables rsyncable mode. */
} ZSTDMT_parameter; } ZSTDMT_parameter;
/* ZSTDMT_setMTCtxParameter() : /* ZSTDMT_setMTCtxParameter() :
@@ -135,7 +137,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
/*! ZSTDMT_CCtxParam_setMTCtxParameter() /*! ZSTDMT_CCtxParam_setMTCtxParameter()
* like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */ * like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value);
/*! ZSTDMT_CCtxParam_setNbWorkers() /*! ZSTDMT_CCtxParam_setNbWorkers()
* Set nbWorkers, and clamp it. * Set nbWorkers, and clamp it.
tests/zstreamtest.c
+1 -1
View File
@@ -1547,7 +1547,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest,
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapLog, FUZ_rand(&lseed) % 12) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */ CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} } } } } }