ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1444 from facebook/btultra2

Browse Source 
btultra2
This commit is contained in:
Yann Collet
2018-12-12 21:09:01 -08:00
committed by GitHub
parent 54ca4b3c5d 9a92ed401d
commit fbcae274a4
21 changed files with 855 additions and 575 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2 -1
View File
@@ -64,7 +64,8 @@ zlibwrapper: lib
## test: run long-duration tests
.PHONY: test
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
DEBUGLEVEL ?= 1
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
test:
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@
doc/zstd_manual.html
+5 -5
View File
@@ -399,10 +399,10 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_lazy2=5,
ZSTD_btlazy2=6,
ZSTD_btopt=7,
ZSTD_btultra=8
</b>/* note : new strategies might be added in the future.<b>
Only the order (from fast to strong) is guaranteed, not the exact position.
new strategy names might be introduced, pushing the maximum number upward */
ZSTD_btultra=8,
ZSTD_btultra2=9
</b>/* note : new strategies _might_ be added in the future.<b>
Only the order (from fast to strong) is guaranteed */
} ZSTD_strategy;
</b></pre><BR>
<pre><b>typedef enum {
@@ -452,7 +452,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Distance between match sampling.
* Larger values make compression faster, and weaker.
* Special: value 0 means "use default targetLength". */
ZSTD_c_compressionStrategy=107, </b>/* See ZSTD_strategy enum definition.<b>
ZSTD_c_strategy=107, </b>/* See ZSTD_strategy enum definition.<b>
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */
lib/compress/zstd_compress.c
+160 -127
View File
@@ -267,9 +267,9 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = ZSTD_TARGETLENGTH_MAX;
return bounds;
case ZSTD_c_compressionStrategy:
bounds.lowerBound = (int)ZSTD_fast;
bounds.upperBound = (int)ZSTD_btultra; /* note : how to ensure at compile time that this is the highest value strategy ? */
case ZSTD_c_strategy:
bounds.lowerBound = ZSTD_STRATEGY_MIN;
bounds.upperBound = ZSTD_STRATEGY_MAX;
return bounds;
case ZSTD_c_contentSizeFlag:
@@ -347,14 +347,15 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
return bounds;
case ZSTD_c_format:
ZSTD_STATIC_ASSERT((int)ZSTD_f_zstd1 < (int)ZSTD_f_zstd1_magicless);
bounds.lowerBound = (int)ZSTD_f_zstd1;
bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
bounds.lowerBound = ZSTD_f_zstd1;
bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds;
case ZSTD_c_forceAttachDict:
bounds.lowerBound = 0;
bounds.upperBound = 1;
ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy);
bounds.lowerBound = ZSTD_dictDefaultAttach;
bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds;
default:
@@ -364,9 +365,21 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
}
}
#define CLAMPCHECK(val,min,max) { \
if (((val)<(min)) | ((val)>(max))) { \
return ERROR(parameter_outOfBound); \
/* ZSTD_cParam_withinBounds:
* @return 1 if value is within cParam bounds,
* 0 otherwise */
static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
{
ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
if (ZSTD_isError(bounds.error)) return 0;
if (value < bounds.lowerBound) return 0;
if (value > bounds.upperBound) return 0;
return 1;
}
#define BOUNDCHECK(cParam, val) { \
if (!ZSTD_cParam_withinBounds(cParam,val)) { \
return ERROR(parameter_outOfBound); \
} }
@@ -380,7 +393,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_searchLog:
case ZSTD_c_minMatch:
case ZSTD_c_targetLength:
case ZSTD_c_compressionStrategy:
case ZSTD_c_strategy:
return 1;
case ZSTD_c_format:
@@ -429,7 +442,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_searchLog:
case ZSTD_c_minMatch:
case ZSTD_c_targetLength:
case ZSTD_c_compressionStrategy:
case ZSTD_c_strategy:
if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@@ -476,14 +489,14 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
switch(param)
{
case ZSTD_c_format :
if (value > (int)ZSTD_f_zstd1_magicless)
return ERROR(parameter_unsupported);
BOUNDCHECK(ZSTD_c_format, value);
CCtxParams->format = (ZSTD_format_e)value;
return (size_t)CCtxParams->format;
case ZSTD_c_compressionLevel : {
int cLevel = value;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevel < ZSTD_minCLevel()) cLevel = ZSTD_minCLevel();
if (cLevel) { /* 0 : does not change current level */
CCtxParams->compressionLevel = cLevel;
}
@@ -493,42 +506,42 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_windowLog :
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
BOUNDCHECK(ZSTD_c_windowLog, value);
CCtxParams->cParams.windowLog = value;
return CCtxParams->cParams.windowLog;
case ZSTD_c_hashLog :
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
BOUNDCHECK(ZSTD_c_hashLog, value);
CCtxParams->cParams.hashLog = value;
return CCtxParams->cParams.hashLog;
case ZSTD_c_chainLog :
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
BOUNDCHECK(ZSTD_c_chainLog, value);
CCtxParams->cParams.chainLog = value;
return CCtxParams->cParams.chainLog;
case ZSTD_c_searchLog :
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
BOUNDCHECK(ZSTD_c_searchLog, value);
CCtxParams->cParams.searchLog = value;
return value;
case ZSTD_c_minMatch :
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
BOUNDCHECK(ZSTD_c_minMatch, value);
CCtxParams->cParams.minMatch = value;
return CCtxParams->cParams.minMatch;
case ZSTD_c_targetLength :
/* all values are valid. 0 => use default */
BOUNDCHECK(ZSTD_c_targetLength, value);
CCtxParams->cParams.targetLength = value;
return CCtxParams->cParams.targetLength;
case ZSTD_c_compressionStrategy :
case ZSTD_c_strategy :
if (value!=0) /* 0 => use default */
CLAMPCHECK(value, (int)ZSTD_fast, (int)ZSTD_btultra);
BOUNDCHECK(ZSTD_c_strategy, value);
CCtxParams->cParams.strategy = (ZSTD_strategy)value;
return (size_t)CCtxParams->cParams.strategy;
@@ -554,7 +567,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_forceAttachDict : {
const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
CLAMPCHECK(pref, ZSTD_dictDefaultAttach, ZSTD_dictForceCopy);
BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
CCtxParams->attachDictPref = pref;
return CCtxParams->attachDictPref;
}
@@ -594,19 +607,19 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_ldmHashLog :
if (value!=0) /* 0 ==> auto */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
BOUNDCHECK(ZSTD_c_ldmHashLog, value);
CCtxParams->ldmParams.hashLog = value;
return CCtxParams->ldmParams.hashLog;
case ZSTD_c_ldmMinMatch :
if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX);
BOUNDCHECK(ZSTD_c_ldmMinMatch, value);
CCtxParams->ldmParams.minMatchLength = value;
return CCtxParams->ldmParams.minMatchLength;
case ZSTD_c_ldmBucketSizeLog :
if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX);
BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value);
CCtxParams->ldmParams.bucketSizeLog = value;
return CCtxParams->ldmParams.bucketSizeLog;
@@ -654,7 +667,7 @@ size_t ZSTD_CCtxParam_getParameter(
case ZSTD_c_targetLength :
*value = CCtxParams->cParams.targetLength;
break;
case ZSTD_c_compressionStrategy :
case ZSTD_c_strategy :
*value = (unsigned)CCtxParams->cParams.strategy;
break;
case ZSTD_c_contentSizeFlag :
@@ -832,16 +845,13 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
@return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CLAMPCHECK(cParams.minMatch, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0);
if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
return ERROR(parameter_outOfBound);
if ((U32)(cParams.strategy) > (U32)ZSTD_btultra)
return ERROR(parameter_unsupported);
BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog);
BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog);
BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog);
BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog);
BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch);
BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength);
BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
return 0;
}
@@ -851,19 +861,19 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
static ZSTD_compressionParameters
ZSTD_clampCParams(ZSTD_compressionParameters cParams)
{
# define CLAMP(val,min,max) { \
if (val<min) val=min; \
else if (val>max) val=max; \
# define CLAMP_TYPE(cParam, val, type) { \
ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \
if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \
else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
}
CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CLAMP(cParams.minMatch, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX);
ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0);
if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
cParams.targetLength = ZSTD_TARGETLENGTH_MAX;
CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra);
# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int)
CLAMP(ZSTD_c_windowLog, cParams.windowLog);
CLAMP(ZSTD_c_chainLog, cParams.chainLog);
CLAMP(ZSTD_c_hashLog, cParams.hashLog);
CLAMP(ZSTD_c_searchLog, cParams.searchLog);
CLAMP(ZSTD_c_minMatch, cParams.minMatch);
CLAMP(ZSTD_c_targetLength,cParams.targetLength);
CLAMP_TYPE(ZSTD_c_strategy, cParams.strategy, ZSTD_strategy);
return cParams;
}
@@ -951,8 +961,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32)
+ (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t));
size_t const optSpace = (forCCtx && ((cParams->strategy == ZSTD_btopt) ||
(cParams->strategy == ZSTD_btultra)))
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
? optPotentialSpace
: 0;
DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
@@ -1253,7 +1262,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ZSTD_invalidateMatchState(ms);
/* opt parser space */
if (forCCtx && ((cParams->strategy == ZSTD_btopt) | (cParams->strategy == ZSTD_btultra))) {
if (forCCtx && (cParams->strategy >= ZSTD_btopt)) {
DEBUGLOG(4, "reserving optimal parser space");
ms->opt.litFreq = (U32*)ptr;
ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits);
@@ -1465,16 +1474,17 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
* dictionary tables into the working context is faster than using them
* in-place.
*/
static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = {
8 KB, /* unused */
8 KB, /* ZSTD_fast */
static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = {
8 KB, /* unused */
8 KB, /* ZSTD_fast */
16 KB, /* ZSTD_dfast */
32 KB, /* ZSTD_greedy */
32 KB, /* ZSTD_lazy */
32 KB, /* ZSTD_lazy2 */
32 KB, /* ZSTD_btlazy2 */
32 KB, /* ZSTD_btopt */
8 KB /* ZSTD_btultra */
8 KB, /* ZSTD_btultra */
8 KB /* ZSTD_btultra2 */
};
static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict,
@@ -1829,7 +1839,9 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
* note : use same formula for both situations */
static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
{
U32 const minlog = (strat==ZSTD_btultra) ? 7 : 6;
U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
return (srcSize >> minlog) + 2;
}
@@ -2423,7 +2435,11 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table");
nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode;
LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, count, max, mostFrequent, nbSeq, LLFSELog, prevEntropy->fse.litlengthCTable, LL_defaultNorm, LL_defaultNormLog, ZSTD_defaultAllowed, strategy);
LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode,
count, max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->fse.litlengthCTable,
LL_defaultNorm, LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
@@ -2442,7 +2458,11 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode;
Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, count, max, mostFrequent, nbSeq, OffFSELog, prevEntropy->fse.offcodeCTable, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy);
Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode,
count, max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->fse.offcodeCTable,
OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy);
assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
@@ -2458,7 +2478,11 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode;
MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, count, max, mostFrequent, nbSeq, MLFSELog, prevEntropy->fse.matchlengthCTable, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy);
MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode,
count, max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->fse.matchlengthCTable,
ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
@@ -2536,7 +2560,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
* assumption : strat is a valid strategy */
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode)
{
static const ZSTD_blockCompressor blockCompressor[3][(unsigned)ZSTD_btultra+1] = {
static const ZSTD_blockCompressor blockCompressor[3][ZSTD_STRATEGY_MAX+1] = {
{ ZSTD_compressBlock_fast /* default for 0 */,
ZSTD_compressBlock_fast,
ZSTD_compressBlock_doubleFast,
@@ -2545,7 +2569,8 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_compressBlock_lazy2,
ZSTD_compressBlock_btlazy2,
ZSTD_compressBlock_btopt,
ZSTD_compressBlock_btultra },
ZSTD_compressBlock_btultra,
ZSTD_compressBlock_btultra2 },
{ ZSTD_compressBlock_fast_extDict /* default for 0 */,
ZSTD_compressBlock_fast_extDict,
ZSTD_compressBlock_doubleFast_extDict,
@@ -2554,6 +2579,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_compressBlock_lazy2_extDict,
ZSTD_compressBlock_btlazy2_extDict,
ZSTD_compressBlock_btopt_extDict,
ZSTD_compressBlock_btultra_extDict,
ZSTD_compressBlock_btultra_extDict },
{ ZSTD_compressBlock_fast_dictMatchState /* default for 0 */,
ZSTD_compressBlock_fast_dictMatchState,
@@ -2563,14 +2589,14 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_compressBlock_lazy2_dictMatchState,
ZSTD_compressBlock_btlazy2_dictMatchState,
ZSTD_compressBlock_btopt_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState }
};
ZSTD_blockCompressor selectedCompressor;
ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
assert((U32)strat >= (U32)ZSTD_fast);
assert((U32)strat <= (U32)ZSTD_btultra);
selectedCompressor = blockCompressor[(int)dictMode][(U32)strat];
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
assert(selectedCompressor != NULL);
return selectedCompressor;
}
@@ -2967,6 +2993,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
if (srcSize >= HASH_READ_SIZE)
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
break;
@@ -3037,7 +3064,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
/* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
/* fill all offset symbols to avoid garbage at end of table */
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable, offcodeNCount, MaxOff, offcodeLog, workspace, HUF_WORKSPACE_SIZE),
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable,
offcodeNCount, MaxOff, offcodeLog,
workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
@@ -3049,7 +3078,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
/* Every match length code must have non-zero probability */
CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE),
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable,
matchlengthNCount, matchlengthMaxValue, matchlengthLog,
workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
@@ -3061,7 +3092,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
/* Every literal length code must have non-zero probability */
CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE),
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable,
litlengthNCount, litlengthMaxValue, litlengthLog,
workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
@@ -4115,27 +4148,27 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
/* W, C, H, S, L, TL, strat */
{ 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
{ 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
{ 20, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */
{ 20, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */
{ 20, 18, 18, 2, 5, 2, ZSTD_greedy }, /* level 5 */
{ 21, 18, 19, 2, 5, 4, ZSTD_lazy }, /* level 6 */
{ 21, 18, 19, 3, 5, 8, ZSTD_lazy2 }, /* level 7 */
{ 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
{ 21, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */
{ 21, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */
{ 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */
{ 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */
{ 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */
{ 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */
{ 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */
{ 21, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */
{ 21, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */
{ 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */
{ 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
{ 22, 22, 22, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
{ 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */
{ 23, 22, 22, 4, 4, 64, ZSTD_btopt }, /* level 17 */
{ 23, 23, 22, 6, 3,256, ZSTD_btopt }, /* level 18 */
{ 23, 24, 22, 7, 3,256, ZSTD_btultra }, /* level 19 */
{ 25, 25, 23, 7, 3,256, ZSTD_btultra }, /* level 20 */
{ 26, 26, 24, 7, 3,512, ZSTD_btultra }, /* level 21 */
{ 27, 27, 25, 9, 3,999, ZSTD_btultra }, /* level 22 */
{ 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */
{ 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */
{ 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */
{ 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
{ 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
{ 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */
{ 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */
{ 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */
{ 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */
{ 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */
{ 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */
{ 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */
},
{ /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */
@@ -4150,18 +4183,18 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 18, 18, 19, 5, 4, 16, ZSTD_btlazy2 }, /* level 11.*/
{ 18, 19, 19, 6, 4, 16, ZSTD_btlazy2 }, /* level 12.*/
{ 18, 19, 19, 8, 4, 16, ZSTD_btlazy2 }, /* level 13 */
{ 18, 18, 19, 4, 4, 24, ZSTD_btopt }, /* level 14.*/
{ 18, 18, 19, 4, 3, 24, ZSTD_btopt }, /* level 15.*/
{ 18, 19, 19, 6, 3, 64, ZSTD_btopt }, /* level 16.*/
{ 18, 19, 19, 8, 3,128, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 19, 10, 3,256, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 19, 10, 3,256, ZSTD_btultra }, /* level 19.*/
{ 18, 19, 19, 11, 3,512, ZSTD_btultra }, /* level 20.*/
{ 18, 19, 19, 12, 3,512, ZSTD_btultra }, /* level 21.*/
{ 18, 19, 19, 13, 3,999, ZSTD_btultra }, /* level 22.*/
{ 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
{ 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
{ 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
{ 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
{ 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/
{ 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/
{ 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/
{ 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/
{ 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
{ 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/
{ 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/
{ 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/
},
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */
@@ -4176,26 +4209,26 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */
{ 17, 18, 17, 6, 4, 16, ZSTD_btlazy2 }, /* level 12 */
{ 17, 18, 17, 8, 4, 16, ZSTD_btlazy2 }, /* level 13.*/
{ 17, 18, 17, 4, 4, 32, ZSTD_btopt }, /* level 14.*/
{ 17, 18, 17, 6, 3, 64, ZSTD_btopt }, /* level 15.*/
{ 17, 18, 17, 7, 3,128, ZSTD_btopt }, /* level 16.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 17.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 18.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 19.*/
{ 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/
{ 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/
{ 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/
{ 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
{ 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
{ 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
{ 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
{ 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/
{ 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/
{ 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/
{ 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/
{ 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/
{ 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
{ 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/
},
{ /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */
{ 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
{ 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
{ 14, 14, 14, 2, 4, 1, ZSTD_dfast }, /* level 3.*/
{ 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4.*/
{ 14, 14, 15, 2, 4, 1, ZSTD_dfast }, /* level 3 */
{ 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
{ 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
{ 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
{ 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
@@ -4203,17 +4236,17 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/
{ 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/
{ 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/
{ 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/
{ 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/
{ 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/
{ 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/
{ 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btopt }, /* level 18.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btultra }, /* level 19.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/
{ 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/
{ 14, 15, 15, 10, 3,512, ZSTD_btultra }, /* level 22.*/
{ 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/
{ 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/
{ 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/
{ 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/
{ 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/
{ 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/
{ 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
{ 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/
{ 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
{ 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/
},
};
@@ -4232,8 +4265,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL;
{ ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */
return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); }
return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize);
}
}
/*! ZSTD_getParams() :
lib/compress/zstd_ldm.c
+1
View File
@@ -188,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
case ZSTD_btlazy2:
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
break;
default:
assert(0); /* not possible : not a valid strategy id */
lib/compress/zstd_opt.c
+54 -20
View File
@@ -1071,39 +1071,69 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1);
}
/* ZSTD_initStats_ultra():
* make a first compression pass, just to seed stats with more accurate starting values.
* only works on first block, with no dictionary and no ldm.
* this function must not fail, hence its usage conditions must be respected.
*/
static void ZSTD_initStats_ultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
DEBUGLOG(5, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
/* invalidate first scan from history */
ZSTD_resetSeqStore(seqStore);
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
}
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
#if 0
/* 2-pass strategy (disabled)
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
/* 2-pass strategy
* this strategy makes a first pass over first block to collect statistics
* and seed next round's statistics with it.
* After 1st pass, function forgets everything, and starts a new block.
* Consequently, this can only work if no data has been previously loaded in tables,
* aka, no dictionary, no prefix, no ldm preprocessing.
* The compression ratio gain is generally small (~0.5% on first block),
* the cost is 2x cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */
U32 tmpRep[ZSTD_REP_NUM];
DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics");
assert(ms->nextToUpdate >= ms->window.dictLimit
&& ms->nextToUpdate <= ms->window.dictLimit + 1);
memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
ZSTD_resetSeqStore(seqStore);
/* invalidate first scan from history */
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
&& (ms->window.dictLimit - ms->nextToUpdate <= 1) /* no prefix (note: intentional overflow, defined as 2-complement) */
) {
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
}
#endif
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
@@ -1134,3 +1164,7 @@ size_t ZSTD_compressBlock_btultra_extDict(
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
}
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
lib/compress/zstd_opt.h
+8
View File
@@ -26,6 +26,10 @@ size_t ZSTD_compressBlock_btopt(
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -41,6 +45,10 @@ size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
#if defined (__cplusplus)
}
#endif
lib/zstd.h
+9 -6
View File
@@ -494,10 +494,10 @@ typedef enum { ZSTD_fast=1,
ZSTD_lazy2=5,
ZSTD_btlazy2=6,
ZSTD_btopt=7,
ZSTD_btultra=8
/* note : new strategies might be added in the future.
Only the order (from fast to strong) is guaranteed, not the exact position.
new strategy names might be introduced, pushing the maximum number upward */
ZSTD_btultra=8,
ZSTD_btultra2=9
/* note : new strategies _might_ be added in the future.
Only the order (from fast to strong) is guaranteed */
} ZSTD_strategy;
@@ -541,14 +541,14 @@ typedef enum {
* , for all strategies > fast, effective maximum is 6.
* Special: value 0 means "use default minMatchLength". */
ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
* For strategies btopt & btultra:
* For strategies btopt, btultra & btultra2:
* Length of Match considered "good enough" to stop search.
* Larger values make compression stronger, and slower.
* For strategy fast:
* Distance between match sampling.
* Larger values make compression faster, and weaker.
* Special: value 0 means "use default targetLength". */
ZSTD_c_compressionStrategy=107, /* See ZSTD_strategy enum definition.
ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */
@@ -947,6 +947,9 @@ ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
#define ZSTD_STRATEGY_MIN ZSTD_fast
#define ZSTD_STRATEGY_MAX ZSTD_btultra2
#define ZSTD_OVERLAPLOG_MIN 0
#define ZSTD_OVERLAPLOG_MAX 9
programs/benchzstd.c
+30 -29
View File
@@ -94,6 +94,18 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return errorNum; \
}
#define CHECK_Z(zf) { \
size_t const zerr = zf; \
if (ZSTD_isError(zerr)) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAY("Error : "); \
DISPLAY("%s failed : %s", \
#zf, ZSTD_getErrorName(zerr)); \
DISPLAY(" \n"); \
exit(1); \
} \
}
#define RETURN_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
@@ -105,17 +117,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return r; \
}
/* error without displaying */
#define RETURN_QUIET_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error %i : ", errorNum); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
r.tag = errorNum; \
return r; \
}
/* *************************************
* Benchmark Parameters
@@ -162,30 +163,30 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
if (adv->nbWorkers==1) {
ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0);
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
} else {
ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers);
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
}
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) {
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize));
}
programs/fileio.c
+1 -1
View File
@@ -551,7 +551,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionStrategy, comprParams.strategy) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
programs/zstd.1
+3 -3
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "November 2018" "zstd 1.3.8" "User Commands"
.TH "ZSTD" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -316,7 +316,7 @@ set process priority to real\-time
Specify a strategy used by a match finder\.
.
.IP
There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra\.
There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
.
.TP
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@@ -373,7 +373,7 @@ The minimum \fImml\fR is 3 and the maximum is 7\.
The impact of this field vary depending on selected strategy\.
.
.IP
For ZSTD_btopt and ZSTD_btultra, it specifies the minimum match length that causes match finder to stop searching for better matches\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
.
.IP
For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\.
programs/zstd.1.md
+6 -5
View File
@@ -339,9 +339,10 @@ The list of available _options_:
- `strategy`=_strat_, `strat`=_strat_:
Specify a strategy used by a match finder.
There are 8 strategies numbered from 1 to 8, from faster to stronger:
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
There are 9 strategies numbered from 1 to 9, from faster to stronger:
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
- `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance.
@@ -394,8 +395,8 @@ The list of available _options_:
- `targetLen`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length
that causes match finder to stop searching for better matches.
For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
the minimum match length that causes match finder to stop searching.
A larger `targetLen` usually improves compression ratio
but decreases compression speed.
programs/zstdgrep.1
+1 -1
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTDGREP" "1" "November 2018" "zstd 1.3.8" "User Commands"
.TH "ZSTDGREP" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
programs/zstdless.1
+1 -1
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTDLESS" "1" "November 2018" "zstd 1.3.8" "User Commands"
.TH "ZSTDLESS" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
tests/fullbench.c
+2 -2
View File
@@ -401,7 +401,7 @@ static size_t benchMem(U32 benchNb,
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionStrategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
@@ -411,7 +411,7 @@ static size_t benchMem(U32 benchNb,
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionStrategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
/* Preparation */
switch(benchNb)
tests/fuzz/zstd_helpers.c
+3 -3
View File
@@ -13,7 +13,7 @@
#include "fuzz_helpers.h"
#include "zstd.h"
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value)
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
{
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
}
@@ -35,7 +35,7 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
ZSTD_MINMATCH_MAX);
cParams.targetLength = FUZZ_rand32(state, 0, 512);
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX);
return ZSTD_adjustCParams(cParams, srcSize, 0);
}
@@ -66,7 +66,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
set(cctx, ZSTD_c_searchLog, cParams.searchLog);
set(cctx, ZSTD_c_minMatch, cParams.minMatch);
set(cctx, ZSTD_c_targetLength, cParams.targetLength);
set(cctx, ZSTD_c_compressionStrategy, cParams.strategy);
set(cctx, ZSTD_c_strategy, cParams.strategy);
/* Select frame parameters */
setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
tests/paramgrill.c
+431 -239
View File
@@ -75,20 +75,21 @@ static const int g_maxNbVariations = 64;
#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
#define MML_RANGE (ZSTD_MINMATCH_MAX - ZSTD_MINMATCH_MIN + 1)
#define TLEN_RANGE 17
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
#define FADT_RANGE 3
#define MML_RANGE (ZSTD_MINMATCH_MAX - ZSTD_MINMATCH_MIN + 1)
#define TLEN_RANGE 17
#define STRT_RANGE (ZSTD_STRATEGY_MAX - ZSTD_STRATEGY_MIN + 1)
#define FADT_RANGE 3
#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
#define CHECKTIME(r) { if(BMK_timeSpan_s(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
#define CHECKTIMEGT(ret, val, _gototag) { if(BMK_timeSpan_s(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */
static const char* g_stratName[ZSTD_btultra+1] = {
static const char* g_stratName[ZSTD_STRATEGY_MAX+1] = {
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra ",
"ZSTD_btultra2"};
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
@@ -114,13 +115,13 @@ typedef struct {
U32 vals[NUM_PARAMS];
} paramValues_t;
/* maximum value of parameters */
static const U32 mintable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_MINMATCH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
/* minimum value of parameters */
static const U32 mintable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_MINMATCH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_STRATEGY_MIN, FADT_MIN };
/* maximum value of parameters */
static const U32 maxtable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_MINMATCH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_MINMATCH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_STRATEGY_MAX, FADT_MAX };
/* # of values parameters can take on */
static const U32 rangetable[NUM_PARAMS] =
@@ -128,7 +129,7 @@ static const U32 rangetable[NUM_PARAMS] =
/* ZSTD_cctxSetParameter() index to set */
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
{ ZSTD_c_windowLog, ZSTD_c_chainLog, ZSTD_c_hashLog, ZSTD_c_searchLog, ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_compressionStrategy, ZSTD_c_forceAttachDict };
{ ZSTD_c_windowLog, ZSTD_c_chainLog, ZSTD_c_hashLog, ZSTD_c_searchLog, ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_strategy, ZSTD_c_forceAttachDict };
/* names of parameters */
static const char* g_paramNames[NUM_PARAMS] =
@@ -139,13 +140,10 @@ static const char* g_shortParamNames[NUM_PARAMS] =
{ "wlog", "clog", "hlog", "slog", "mml", "tlen", "strat", "fadt" };
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
static U32 rangeMap(varInds_t param, int ind) {
static U32 rangeMap(varInds_t param, int ind)
{
ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0);
switch(param) {
case tlen_ind:
return tlen_table[ind];
case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
return ind - 1;
case wlog_ind: /* using default: triggers -Wswitch-enum */
case clog_ind:
case hlog_ind:
@@ -153,17 +151,30 @@ static U32 rangeMap(varInds_t param, int ind) {
case mml_ind:
case strt_ind:
return mintable[param] + ind;
case tlen_ind:
return tlen_table[ind];
case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
return ind - 1;
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n ");
return (U32)-1;
default:;
}
return 0; /* should never happen, stop compiler warnings */
DISPLAY("Error, not a valid param\n ");
assert(0);
return (U32)-1;
}
/* inverse of rangeMap */
static int invRangeMap(varInds_t param, U32 value) {
static int invRangeMap(varInds_t param, U32 value)
{
value = MIN(MAX(mintable[param], value), maxtable[param]);
switch(param) {
case wlog_ind:
case clog_ind:
case hlog_ind:
case slog_ind:
case mml_ind:
case strt_ind:
return value - mintable[param];
case tlen_ind: /* bin search */
{
int lo = 0;
@@ -182,33 +193,49 @@ static int invRangeMap(varInds_t param, U32 value) {
}
case fadt_ind:
return (int)value + 1;
case wlog_ind:
case clog_ind:
case hlog_ind:
case slog_ind:
case mml_ind:
case strt_ind:
return value - mintable[param];
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n ");
return -2;
default:;
}
return 0; /* should never happen, stop compiler warnings */
DISPLAY("Error, not a valid param\n ");
assert(0);
return -2;
}
/* display of params */
static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
static void displayParamVal(FILE* f, varInds_t param, U32 value, int width)
{
switch(param) {
case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
case wlog_ind:
case clog_ind:
case hlog_ind:
case slog_ind:
case mml_ind:
case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
case tlen_ind:
if(width) {
fprintf(f, "%*u", width, value);
} else {
fprintf(f, "%u", value);
}
break;
case strt_ind:
if(width) {
fprintf(f, "%*s", width, g_stratName[value]);
} else {
fprintf(f, "%s", g_stratName[value]);
}
break;
case fadt_ind: /* force attach dict */
if(width) {
fprintf(f, "%*d", width, (int)value);
} else {
fprintf(f, "%d", (int)value);
}
break;
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n "); break;
default:
DISPLAY("Error, not a valid param\n ");
assert(0);
break;
}
}
@@ -217,8 +244,6 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
* Benchmark Parameters/Global Variables
**************************************/
typedef BYTE U8;
/* General Utility */
static U32 g_timeLimit_s = 99999; /* about 27 hours */
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
@@ -227,7 +252,7 @@ static U32 g_rand = 1;
/* Display */
static int g_displayLevel = 3;
static BYTE g_silenceParams[NUM_PARAMS];
static BYTE g_silenceParams[NUM_PARAMS]; /* can selectively silence some params when displaying them */
/* Mode Selection */
static U32 g_singleRun = 0;
@@ -298,13 +323,14 @@ static paramValues_t sanitizeParams(paramValues_t params)
params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
if (params.vals[strt_ind] == ZSTD_dfast)
params.vals[slog_ind] = 0;
if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
if ( (params.vals[strt_ind] < ZSTD_btopt) && (params.vals[strt_ind] != ZSTD_fast) )
params.vals[tlen_ind] = 0;
return params;
}
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p)
{
ZSTD_compressionParameters c;
memset(&c, 0, sizeof(ZSTD_compressionParameters));
c.windowLog = p.vals[wlog_ind];
@@ -318,7 +344,8 @@ static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
return c;
}
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c)
{
paramValues_t p;
varInds_t i;
p.vals[wlog_ind] = c.windowLog;
@@ -330,18 +357,20 @@ static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
p.vals[strt_ind] = c.strategy;
/* set all other params to their minimum value */
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
for (i = strt_ind + 1; i < NUM_PARAMS; i++) {
p.vals[i] = mintable[i];
}
return p;
}
/* equivalent of ZSTD_adjustCParams for paramValues_t */
static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) {
static paramValues_t
adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize)
{
paramValues_t ot = p;
varInds_t i;
p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
if(!dictSize) { p.vals[fadt_ind] = 0; }
if (!dictSize) { p.vals[fadt_ind] = 0; }
/* retain value of all other parameters */
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
p.vals[i] = ot.vals[i];
@@ -368,7 +397,10 @@ static size_t BMK_findMaxMem(U64 requiredMem)
}
/* accuracy in seconds only, span can be multiple years */
static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
static U32 BMK_timeSpan_s(const UTIL_time_t tStart)
{
return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL);
}
static U32 FUZ_rotl32(U32 x, U32 r)
{
@@ -387,32 +419,38 @@ static U32 FUZ_rand(U32* src)
return rand32 >> 5;
}
/* allows zeros */
#define CLAMPCHECK(val,min,max) { \
#define BOUNDCHECK(val,min,max) { \
if (((val)<(min)) | ((val)>(max))) { \
DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \
return 0; \
} }
static int paramValid(const paramValues_t paramTarget) {
static int paramValid(const paramValues_t paramTarget)
{
U32 i;
for(i = 0; i < NUM_PARAMS; i++) {
CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
BOUNDCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
}
return 1;
}
static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
varInds_t i;
for(i = 0; i < NUM_PARAMS; i++) {
if(paramTarget.vals[i] == PARAM_UNSET) {
paramTarget.vals[i] = mintable[i];
/* cParamUnsetMin() :
* if any parameter in paramTarget is not yet set,
* it will receive its corresponding minimal value.
* This function never fails */
static paramValues_t cParamUnsetMin(paramValues_t paramTarget)
{
varInds_t vi;
for (vi = 0; vi < NUM_PARAMS; vi++) {
if (paramTarget.vals[vi] == PARAM_UNSET) {
paramTarget.vals[vi] = mintable[vi];
}
}
return paramTarget;
}
static paramValues_t emptyParams(void) {
static paramValues_t emptyParams(void)
{
U32 i;
paramValues_t p;
for(i = 0; i < NUM_PARAMS; i++) {
@@ -421,7 +459,8 @@ static paramValues_t emptyParams(void) {
return p;
}
static winnerInfo_t initWinnerInfo(const paramValues_t p) {
static winnerInfo_t initWinnerInfo(const paramValues_t p)
{
winnerInfo_t w1;
w1.result.cSpeed = 0.;
w1.result.dSpeed = 0.;
@@ -431,7 +470,9 @@ static winnerInfo_t initWinnerInfo(const paramValues_t p) {
return w1;
}
static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
static paramValues_t
overwriteParams(paramValues_t base, const paramValues_t mask)
{
U32 i;
for(i = 0; i < NUM_PARAMS; i++) {
if(mask.vals[i] != PARAM_UNSET) {
@@ -441,12 +482,16 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas
return base;
}
static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
static void
paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr)
{
ptr->vals[paramIndex] = rangeMap(paramIndex,
invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
}
/* varies ptr by nbChanges respecting varyParams*/
static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
static void
paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
{
paramValues_t p;
U32 validated = 0;
@@ -474,8 +519,9 @@ static paramValues_t randomParams(void)
static U64 g_clockGranularity = 100000000ULL;
static void findClockGranularity(void) {
UTIL_time_t clockStart = UTIL_getTime();
static void init_clockGranularity(void)
{
UTIL_time_t const clockStart = UTIL_getTime();
U64 el1 = 0, el2 = 0;
int i = 0;
do {
@@ -512,7 +558,9 @@ static int feasible(const BMK_benchResult_t results, const constraint_t target)
* bonus to exceeding the constraint value. We also give linear ratio for compression ratio.
* The constant factors are experimental.
*/
static double resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_t target) {
static double
resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_t target)
{
double cs = 0., ds = 0., rt, cm = 0.;
const double r1 = 1, r2 = 0.1, rtr = 0.5;
double ret;
@@ -528,7 +576,9 @@ static double resultScore(const BMK_benchResult_t res, const size_t srcSize, con
}
/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
static double resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes) {
static double
resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes)
{
double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
@@ -538,16 +588,21 @@ static double resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResu
}
/* return true if r2 strictly better than r1 */
static int compareResultLT(const BMK_benchResult_t result1, const BMK_benchResult_t result2, const constraint_t target, size_t srcSize) {
static int
compareResultLT(const BMK_benchResult_t result1, const BMK_benchResult_t result2, const constraint_t target, size_t srcSize)
{
if(feasible(result1, target) && feasible(result2, target)) {
if(g_optmode) {
return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
} else {
return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
|| (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
return (result1.cSize > result2.cSize)
|| (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
|| (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
}
}
return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
return feasible(result2, target)
|| (!feasible(result1, target)
&& (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
}
static constraint_t relaxTarget(constraint_t target) {
@@ -557,14 +612,17 @@ static constraint_t relaxTarget(constraint_t target) {
return target;
}
static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize)
{
varInds_t v;
for(v = 0; v < NUM_PARAMS; v++) {
if(pc->vals[v] != PARAM_UNSET) {
U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
if(newval != pc->vals[v]) {
pc->vals[v] = newval;
DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
DISPLAY("Warning: parameter %s not in valid range, adjusting to ",
g_paramNames[v]);
displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
}
}
}
@@ -578,7 +636,8 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
U32 adjust = MAX(mintable[wlog_ind], sshb);
if(adjust != pc->vals[wlog_ind]) {
pc->vals[wlog_ind] = adjust;
DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n",
pc->vals[wlog_ind]);
}
}
}
@@ -593,40 +652,52 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
if(pc->vals[clog_ind] > maxclog) {
pc->vals[clog_ind] = maxclog;
DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n",
pc->vals[clog_ind]);
}
}
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n",
pc->vals[hlog_ind]);
}
}
if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
pc->vals[clog_ind] = pc->vals[slog_ind];
DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n",
pc->vals[slog_ind]);
}
}
}
static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
static int
redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize)
{
return
(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
|| ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
|| (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
|| (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
|| (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
}
/*-************************************
* Display Functions
**************************************/
static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
/* BMK_paramValues_into_commandLine() :
* transform a set of parameters paramValues_t
* into a command line compatible with `zstd` syntax
* and writes it into FILE* f.
* f must be already opened and writable */
static void
BMK_paramValues_into_commandLine(FILE* f, const paramValues_t params)
{
varInds_t v;
int first = 1;
fprintf(f,"--zstd=");
@@ -642,53 +713,6 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
fprintf(f, "\n");
}
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize)
{
varInds_t v;
int first = 1;
res.params = cParamUnsetMin(res.params);
fprintf(f, " {");
for (v = 0; v < NUM_PARAMS; v++) {
if (g_silenceParams[v]) { continue; }
if (!first) { fprintf(f, ","); }
displayParamVal(f, v, res.params.vals[v], 3);
first = 0;
}
{ double const ratio = res.result.cSize ?
(double)srcSize / res.result.cSize : 0;
double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT;
double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT;
fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
ratio, cSpeedMBps, dSpeedMBps);
}
}
/* Writes to f the results of a parameter benchmark */
/* when used with --optimize, will only print results better than previously discovered */
static void BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t result, const paramValues_t params, const size_t srcSize)
{
char lvlstr[15] = "Custom Level";
winnerInfo_t w;
w.params = params;
w.result = result;
fprintf(f, "\r%79s\r", "");
if(cLevel != CUSTOM_LEVEL) {
snprintf(lvlstr, 15, " Level %2d ", cLevel);
}
if(TIMED) {
const U64 time = UTIL_clockSpanNano(g_time);
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
}
fprintf(f, "/* %s */ ", lvlstr);
BMK_displayOneResult(f, w, srcSize);
}
/* comparison function: */
/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
@@ -699,7 +723,9 @@ static void BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t r
#define SPEED_RESULT 4
#define SIZE_RESULT 5
/* maybe have epsilon-eq to limit table size? */
static int speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t r2) {
static int
speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t r2)
{
if(r1.cSpeed < r2.cSpeed) {
if(r1.cSize >= r2.cSize) {
return BETTER_RESULT;
@@ -715,7 +741,9 @@ static int speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t
/* 0 for insertion, 1 for no insert */
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
static int
insertWinner(const winnerInfo_t w, const constraint_t targetConstraints)
{
BMK_benchResult_t r = w.result;
winner_ll_node* cur_node = g_winners;
/* first node to insert */
@@ -809,20 +837,82 @@ static int insertWinner(const winnerInfo_t w, const constraint_t targetConstrain
}
}
static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
static void
BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize)
{
varInds_t v;
int first = 1;
res.params = cParamUnsetMin(res.params);
fprintf(f, " {");
for (v = 0; v < NUM_PARAMS; v++) {
if (g_silenceParams[v]) { continue; }
if (!first) { fprintf(f, ","); }
displayParamVal(f, v, res.params.vals[v], 3);
first = 0;
}
{ double const ratio = res.result.cSize ?
(double)srcSize / res.result.cSize : 0;
double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT;
double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT;
fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
ratio, cSpeedMBps, dSpeedMBps);
}
}
/* Writes to f the results of a parameter benchmark */
/* when used with --optimize, will only print results better than previously discovered */
static void
BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t result, const paramValues_t params, const size_t srcSize)
{
char lvlstr[15] = "Custom Level";
winnerInfo_t w;
w.params = params;
w.result = result;
fprintf(f, "\r%79s\r", "");
if(cLevel != CUSTOM_LEVEL) {
snprintf(lvlstr, 15, " Level %2d ", cLevel);
}
if(TIMED) {
const U64 mn_in_ns = 60ULL * TIMELOOP_NANOSEC;
const U64 time = UTIL_clockSpanNano(g_time);
const U64 minutes = time / mn_in_ns;
fprintf(f, "%1lu:%2lu:%05.2f - ",
(unsigned long) minutes / 60,
(unsigned long) minutes % 60,
(double)(time - (minutes * mn_in_ns)) / TIMELOOP_NANOSEC );
}
fprintf(f, "/* %s */ ", lvlstr);
BMK_displayOneResult(f, w, srcSize);
}
static void
BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
{
/* global winner used for constraints */
/* cSize, cSpeed, dSpeed, cMem */
static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) {
if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL },
{ { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } }
};
if ( DEBUG
|| compareResultLT(g_winner.result, result, targetConstraints, srcSize)
|| g_displayLevel >= 4) {
if ( DEBUG
&& compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
DISPLAY("New Winner: \n");
}
if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); }
if(g_displayLevel >= 2) {
BMK_printWinner(f, cLevel, result, params, srcSize);
}
if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); }
if(g_displayLevel >= 1) { BMK_paramValues_into_commandLine(f, params); }
g_winner.result = result;
g_winner.params = params;
}
@@ -850,31 +940,88 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_
fprintf(f, "Overall Winner: \n");
BMK_displayOneResult(f, g_winner, srcSize);
BMK_translateAdvancedParams(f, g_winner.params);
BMK_paramValues_into_commandLine(f, g_winner.params);
fprintf(f, "Latest BMK: \n");\
BMK_displayOneResult(f, w, srcSize);
}
}
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
/* BMK_print_cLevelEntry() :
* Writes one cLevelTable entry, for one level.
* f must exist, be already opened, and be seekable.
* this function cannot error.
*/
static void
BMK_print_cLevelEntry(FILE* f, const int cLevel,
paramValues_t params,
const BMK_benchResult_t result, const size_t srcSize)
{
varInds_t v;
int first = 1;
assert(cLevel >= 0);
assert(cLevel <= NB_LEVELS_TRACKED);
params = cParamUnsetMin(params);
fprintf(f, " {");
/* print cParams.
* assumption : all cParams are present and in order in the following range */
for (v = 0; v <= strt_ind; v++) {
if (!first) { fprintf(f, ","); }
displayParamVal(f, v, params.vals[v], 3);
first = 0;
}
/* print comment */
{ double const ratio = result.cSize ?
(double)srcSize / result.cSize : 0;
double const cSpeedMBps = (double)result.cSpeed / MB_UNIT;
double const dSpeedMBps = (double)result.dSpeed / MB_UNIT;
fprintf(f, " }, /* level %2i: R=%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
cLevel, ratio, cSpeedMBps, dSpeedMBps);
}
}
/* BMK_print_cLevelTable() :
* print candidate compression table into proposed FILE* f.
* f must exist, be already opened, and be seekable.
* winners must be a table of NB_LEVELS_TRACKED+1 elements winnerInfo_t, all entries presumed initialized
* this function cannot error.
*/
static void
BMK_print_cLevelTable(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
{
int cLevel;
fprintf(f, "\n /* Proposed configurations : */ \n");
fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++)
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
BMK_print_cLevelEntry(f,
cLevel, winners[cLevel].params,
winners[cLevel].result, srcSize);
}
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
/* BMK_saveAndPrint_cLevelTable() :
* save candidate compression table into FILE* f,
* and then to stdout.
* f must exist, be already opened, and be seekable.
* winners must be a table of NB_LEVELS_TRACKED+1 elements winnerInfo_t, all entries presumed initialized
* this function cannot error.
*/
static void
BMK_saveAndPrint_cLevelTable(FILE* const f,
const winnerInfo_t* winners,
const size_t srcSize)
{
fseek(f, 0, SEEK_SET);
BMK_printWinners2(f, winners, srcSize);
BMK_print_cLevelTable(f, winners, srcSize);
fflush(f);
BMK_printWinners2(stdout, winners, srcSize);
BMK_print_cLevelTable(stdout, winners, srcSize);
}
@@ -1218,7 +1365,7 @@ static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const
if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
|| (varArray[i] == slog_ind && strat == ZSTD_fast)
|| (varArray[i] == slog_ind && strat == ZSTD_dfast)
|| (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
|| (varArray[i] == tlen_ind && strat < ZSTD_btopt && strat != ZSTD_fast))) {
varNew[j] = varArray[i];
j++;
}
@@ -1290,10 +1437,12 @@ static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t
}
/* frees all allocated memotables */
/* secret contract :
* mtAll is a table of (ZSTD_STRATEGY_MAX+1) memoTable_t */
static void freeMemoTableArray(memoTable_t* const mtAll) {
int i;
if(mtAll == NULL) { return; }
for(i = 1; i <= (int)ZSTD_btultra; i++) {
for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) {
free(mtAll[i].table);
}
free(mtAll);
@@ -1301,21 +1450,26 @@ static void freeMemoTableArray(memoTable_t* const mtAll) {
/* inits memotables for all (including mallocs), all strategies */
/* takes unsanitized varyParams */
static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
static memoTable_t*
createMemoTableArray(const paramValues_t p,
const varInds_t* const varyParams,
const size_t varyLen,
const U32 memoTableLog)
{
memoTable_t* const mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_STRATEGY_MAX + 1));
ZSTD_strategy i, stratMin = ZSTD_STRATEGY_MIN, stratMax = ZSTD_STRATEGY_MAX;
if(mtAll == NULL) {
return NULL;
}
for(i = 1; i <= (int)ZSTD_btultra; i++) {
for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) {
mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
}
/* no memoization */
if(memoTableLog == 0) {
for(i = 1; i <= (int)ZSTD_btultra; i++) {
for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) {
mtAll[i].tableType = noMemo;
mtAll[i].table = NULL;
mtAll[i].tableLen = 0;
@@ -1374,6 +1528,16 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
* Benchmarking Functions
**************************************/
static void display_params_tested(paramValues_t cParams)
{
varInds_t vi;
DISPLAYLEVEL(3, "\r testing :");
for (vi=0; vi < NUM_PARAMS; vi++) {
DISPLAYLEVEL(3, "%3u,", cParams.vals[vi]);
}
DISPLAYLEVEL(3, "\b \r");
}
/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
@@ -1404,6 +1568,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
ZSTD_DCtx* dctx = ctx.dctx;
/* init */
display_params_tested(*comprParams);
memset(&bResult, 0, sizeof(bResult));
/* warmimg up memory */
@@ -1511,6 +1676,10 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
}
}
/* BMK_benchParam() :
* benchmark a set of `cParams` over sample `buf`,
* store the result in `resultPtr`.
* @return : 0 if success, 1 if error */
static int BMK_benchParam ( BMK_benchResult_t* resultPtr,
buffers_t buf, contexts_t ctx,
paramValues_t cParams)
@@ -1518,30 +1687,12 @@ static int BMK_benchParam ( BMK_benchResult_t* resultPtr,
BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx,
BASE_CLEVEL, &cParams,
BMK_both, 3);
int const success = BMK_isSuccessful_benchOutcome(outcome);
if (!success) return 1;
if (!BMK_isSuccessful_benchOutcome(outcome)) return 1;
*resultPtr = BMK_extract_benchResult(outcome);
return 0;
}
#define CBENCHMARK(conditional, resultvar, tmpret, mode, sec) { \
if(conditional) { \
BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, sec); \
if (!BMK_isSuccessful_benchOutcome(outcome)) { \
DEBUGOUTPUT("Benchmarking failed\n"); \
return ERROR_RESULT; \
} \
{ BMK_benchResult_t const tmpResult = BMK_extract_benchResult(outcome); \
if (mode != BMK_decodeOnly) { \
resultvar.cSpeed = tmpResult.cSpeed; \
resultvar.cSize = tmpResult.cSize; \
resultvar.cMem = tmpResult.cMem; \
} \
if (mode != BMK_compressOnly) { resultvar.dSpeed = tmpResult.dSpeed; } \
} } \
}
/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
#define VARIANCE 1.2
static int allBench(BMK_benchResult_t* resultPtr,
@@ -1661,12 +1812,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
g_level_constraint[l].cSpeed_min = (g_level_constraint[l-1].cSpeed_min * 49) / 64;
g_level_constraint[l].dSpeed_min = 0.;
g_level_constraint[l].windowLog_max = (l<20) ? 23 : l+5; /* only --ultra levels >= 20 can use windowlog > 23 */
g_level_constraint[l].strategy_max = (l<19) ? ZSTD_btopt : ZSTD_btultra; /* level 19 is allowed to use btultra */
g_level_constraint[l].strategy_max = ZSTD_STRATEGY_MAX;
} }
}
static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
const buffers_t buf, const contexts_t ctx)
static int BMK_seed(winnerInfo_t* winners,
const paramValues_t params,
const buffers_t buf,
const contexts_t ctx)
{
BMK_benchResult_t testResult;
int better = 0;
@@ -1674,8 +1827,8 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
BMK_benchParam(&testResult, buf, ctx, params);
for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
if (testResult.cSpeed < g_level_constraint[cLevel].cSpeed_min)
continue; /* not fast enough for this level */
if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min)
@@ -1688,7 +1841,7 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
/* first solution for this cLevel */
winners[cLevel].result = testResult;
winners[cLevel].params = params;
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
BMK_print_cLevelEntry(stdout, cLevel, params, testResult, buf.srcSize);
better = 1;
continue;
}
@@ -1718,7 +1871,7 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
if (W_DMemUsed_note < O_DMemUsed_note) {
/* uses too much Decompression memory for too little benefit */
if (W_ratio > O_ratio)
DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
DISPLAYLEVEL(3, "Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
W_ratio, (double)(W_DMemUsed) / 1024 / 1024,
O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel);
continue;
@@ -1726,34 +1879,38 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
if (W_CMemUsed_note < O_CMemUsed_note) {
/* uses too much memory for compression for too little benefit */
if (W_ratio > O_ratio)
DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
DISPLAYLEVEL(3, "Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n",
W_ratio, (double)(W_CMemUsed) / 1024 / 1024,
O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel);
O_ratio, (double)(O_CMemUsed) / 1024 / 1024,
cLevel);
continue;
}
if (W_CSpeed_note < O_CSpeed_note ) {
/* too large compression speed difference for the compression benefit */
if (W_ratio > O_ratio)
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
DISPLAYLEVEL(3, "Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
W_ratio, (double)testResult.cSpeed / MB_UNIT,
O_ratio, (double)winners[cLevel].result.cSpeed / MB_UNIT, cLevel);
O_ratio, (double)winners[cLevel].result.cSpeed / MB_UNIT,
cLevel);
continue;
}
if (W_DSpeed_note < O_DSpeed_note ) {
/* too large decompression speed difference for the compression benefit */
if (W_ratio > O_ratio)
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
DISPLAYLEVEL(3, "Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
W_ratio, (double)testResult.dSpeed / MB_UNIT,
O_ratio, (double)winners[cLevel].result.dSpeed / MB_UNIT, cLevel);
O_ratio, (double)winners[cLevel].result.dSpeed / MB_UNIT,
cLevel);
continue;
}
if (W_ratio < O_ratio)
DISPLAY("Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", W_ratio, O_ratio, cLevel);
DISPLAYLEVEL(3, "Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n",
W_ratio, O_ratio, cLevel);
winners[cLevel].result = testResult;
winners[cLevel].params = params;
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
BMK_print_cLevelEntry(stdout, cLevel, params, testResult, buf.srcSize);
better = 1;
} }
@@ -1770,58 +1927,75 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
#define PARAMTABLEMASK (PARAMTABLESIZE-1)
static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
static BYTE* NB_TESTS_PLAYED(paramValues_t p) {
ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p));
return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
static BYTE* NB_TESTS_PLAYED(paramValues_t p)
{
ZSTD_compressionParameters const cParams = pvalsToCParams(sanitizeParams(p));
unsigned long long const h64 = XXH64(&cParams, sizeof(cParams), 0);
return &g_alreadyTested[(h64 >> 3) & PARAMTABLEMASK];
}
static void playAround(FILE* f, winnerInfo_t* winners,
static void playAround(FILE* f,
winnerInfo_t* winners,
paramValues_t p,
const buffers_t buf, const contexts_t ctx)
{
int nbVariations = 0, i;
int nbVariations = 0;
UTIL_time_t const clockStart = UTIL_getTime();
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
BYTE* b;
if (nbVariations++ > g_maxNbVariations) break;
do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } }
while(!paramValid(p));
do {
int i;
for(i = 0; i < 4; i++) {
paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1),
((FUZ_rand(&g_rand) & 1) << 1) - 1,
&p);
}
} while (!paramValid(p));
/* exclude faster if already played params */
if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
continue;
/* test */
b = NB_TESTS_PLAYED(p);
(*b)++;
{ BYTE* const b = NB_TESTS_PLAYED(p);
(*b)++;
}
if (!BMK_seed(winners, p, buf, ctx)) continue;
/* improvement found => search more */
BMK_printWinners(f, winners, buf.srcSize);
BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize);
playAround(f, winners, p, buf, ctx);
}
}
static void BMK_selectRandomStart(
FILE* f, winnerInfo_t* winners,
static void
BMK_selectRandomStart( FILE* f,
winnerInfo_t* winners,
const buffers_t buf, const contexts_t ctx)
{
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) {
/* use some random entry */
paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */
buf.srcSize, 0);
buf.srcSize, 0);
playAround(f, winners, p, buf, ctx);
} else {
playAround(f, winners, winners[id].params, buf, ctx);
}
}
static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
/* BMK_generate_cLevelTable() :
* test a large number of configurations
* and distribute them accross compression levels according to speed conditions.
* display and save all intermediate results into rfName = "grillResults.txt".
* the function automatically stops after g_timeLimit_s.
* this function cannot error, it directly exit() in case of problem.
*/
static void BMK_generate_cLevelTable(const buffers_t buf, const contexts_t ctx)
{
paramValues_t params;
winnerInfo_t winners[NB_LEVELS_TRACKED+1];
@@ -1850,17 +2024,17 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0));
BMK_seed(winners, params, buf, ctx);
} }
BMK_printWinners(f, winners, buf.srcSize);
BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize);
/* start tests */
{ const UTIL_time_t grillStart = UTIL_getTime();
do {
BMK_selectRandomStart(f, winners, buf, ctx);
} while (BMK_timeSpan(grillStart) < g_timeLimit_s);
} while (BMK_timeSpan_s(grillStart) < g_timeLimit_s);
}
/* end summary */
BMK_printWinners(f, winners, buf.srcSize);
BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize);
DISPLAY("grillParams operations completed \n");
/* clean up*/
@@ -1872,7 +2046,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
* Single Benchmark Functions
**************************************/
static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) {
static int
benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel)
{
BMK_benchResult_t testResult;
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
@@ -1922,7 +2098,7 @@ static int benchSample(double compressibility, int cLevel)
if(g_singleRun) {
ret = benchOnce(buf, ctx, cLevel);
} else {
BMK_benchFullTable(buf, ctx);
BMK_generate_cLevelTable(buf, ctx);
}
freeBuffers(buf);
@@ -1962,7 +2138,7 @@ static int benchFiles(const char** fileNamesTable, int nbFiles,
if (g_singleRun) {
ret = benchOnce(buf, ctx, cLevel);
} else {
BMK_benchFullTable(buf, ctx);
BMK_generate_cLevelTable(buf, ctx);
}
freeBuffers(buf);
@@ -2039,15 +2215,15 @@ static winnerInfo_t climbOnce(const constraint_t target,
}
}
}
}
} /* for (offset = -1; offset <= 1; offset += 2) */
} /* for (i = 0; i < varLen; i++) */
if(better) {
continue;
}
for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
for (dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
for (i = 0; i < (1 << varLen) / varLen + 2; i++) {
int res;
CHECKTIME(winnerInfo);
candidateInfo.params = cparam;
@@ -2094,11 +2270,12 @@ static winnerInfo_t climbOnce(const constraint_t target,
weight more on visit for bad results, less on good results/more on later results / ones with more failures.
allocate memoTable here.
*/
static winnerInfo_t optimizeFixedStrategy(
const buffers_t buf, const contexts_t ctx,
const constraint_t target, paramValues_t paramTarget,
const ZSTD_strategy strat,
memoTable_t* memoTableArray, const int tries) {
static winnerInfo_t
optimizeFixedStrategy(const buffers_t buf, const contexts_t ctx,
const constraint_t target, paramValues_t paramTarget,
const ZSTD_strategy strat,
memoTable_t* memoTableArray, const int tries)
{
int i = 0;
paramValues_t init;
@@ -2113,9 +2290,11 @@ static winnerInfo_t optimizeFixedStrategy(
for(i = 0; i < tries; i++) {
DEBUGOUTPUT("Restart\n");
do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize));
do {
randomConstrainedParams(&init, memoTableArray, strat);
} while(redundantParams(init, target, buf.maxBlockSize));
candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
if (compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
i = 0;
@@ -2129,12 +2308,13 @@ static winnerInfo_t optimizeFixedStrategy(
/* goes best, best-1, best+1, best-2, ... */
/* return 0 if nothing remaining */
static int nextStrategy(const int currentStrategy, const int bestStrategy) {
static int nextStrategy(const int currentStrategy, const int bestStrategy)
{
if(bestStrategy <= currentStrategy) {
int candidate = 2 * bestStrategy - currentStrategy - 1;
if(candidate < 1) {
candidate = currentStrategy + 1;
if(candidate > (int)ZSTD_btultra) {
if(candidate > (int)ZSTD_STRATEGY_MAX) {
return 0;
} else {
return candidate;
@@ -2144,7 +2324,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
}
} else { /* bestStrategy >= currentStrategy */
int candidate = 2 * bestStrategy - currentStrategy;
if(candidate > (int)ZSTD_btultra) {
if(candidate > (int)ZSTD_STRATEGY_MAX) {
candidate = currentStrategy - 1;
if(candidate < 1) {
return 0;
@@ -2175,8 +2355,12 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
static int g_maxTries = 5;
#define TRY_DECAY 1
static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget,
const int cLevelOpt, const int cLevelRun, const U32 memoTableLog)
static int
optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles,
const char* dictFileName,
constraint_t target, paramValues_t paramTarget,
const int cLevelOpt, const int cLevelRun,
const U32 memoTableLog)
{
varInds_t varArray [NUM_PARAMS];
int ret = 0;
@@ -2188,18 +2372,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
buffers_t buf;
g_time = UTIL_getTime();
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
if (createBuffers(&buf, fileNamesTable, nbFiles)) {
DISPLAY("unable to load files\n");
return 1;
}
if(createContexts(&ctx, dictFileName)) {
if (createContexts(&ctx, dictFileName)) {
DISPLAY("unable to load dictionary\n");
freeBuffers(buf);
return 2;
}
if(nbFiles == 1) {
if (nbFiles == 1) {
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]);
} else {
DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles);
@@ -2278,16 +2462,14 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); }
DISPLAYLEVEL(2, "\n");
findClockGranularity();
init_clockGranularity();
{ paramValues_t CParams;
/* find best solution from default params */
{
/* strategy selection */
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
DEBUGOUTPUT("Strategy Selection\n");
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
if (paramTarget.vals[strt_ind] == PARAM_UNSET) {
BMK_benchResult_t candidate;
int i;
for (i=1; i<=maxSeeds; i++) {
@@ -2312,16 +2494,14 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
DEBUGOUTPUT("Real Opt\n");
/* start 'real' optimization */
{
int bestStrategy = (int)winner.params.vals[strt_ind];
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
{ int bestStrategy = (int)winner.params.vals[strt_ind];
if (paramTarget.vals[strt_ind] == PARAM_UNSET) {
int st = bestStrategy;
int tries = g_maxTries;
{
/* one iterations of hill climbing with the level-defined parameters. */
winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params);
if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
/* one iterations of hill climbing with the level-defined parameters. */
{ winnerInfo_t const w1 = climbOnce(target, allMT, buf, ctx, winner.params);
if (compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
winner = w1;
}
CHECKTIMEGT(ret, 0, _displayCleanUp);
@@ -2355,13 +2535,16 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
DISPLAY("No feasible solution found\n");
goto _cleanUp;
}
/* end summary */
_displayCleanUp:
if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); }
BMK_translateAdvancedParams(stdout, winner.params);
if (g_displayLevel >= 0) {
BMK_displayOneResult(stdout, winner, buf.srcSize);
}
BMK_paramValues_into_commandLine(stdout, winner.params);
DISPLAYLEVEL(1, "grillParams size - optimizer completed \n");
}
_cleanUp:
freeContexts(ctx);
freeBuffers(buf);
@@ -2472,14 +2655,23 @@ static int badusage(const char* exename)
return 1;
}
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { \
if ( longCommandWArg(&argument, stringLong) \
|| longCommandWArg(&argument, stringShort) ) { \
variable = readU32FromChar(&argument); \
if (argument[0]==',') { \
argument++; continue; \
} else break; \
} }
/* 1 if successful parse, 0 otherwise */
static int parse_params(const char** argptr, paramValues_t* pv) {
int matched = 0;
const char* argOrig = *argptr;
varInds_t v;
for(v = 0; v < NUM_PARAMS; v++) {
if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) {
if ( longCommandWArg(argptr,g_shortParamNames[v])
|| longCommandWArg(argptr, g_paramNames[v]) ) {
if(**argptr == '=') {
(*argptr)++;
pv->vals[v] = readU32FromChar(argptr);
tests/playTests.sh
+2 -1
View File
@@ -237,6 +237,7 @@ $ECHO "\n===> Advanced compression parameters "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist
test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
@@ -244,7 +245,7 @@ roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
roundTripTest -g512K 19
roundTripTest -g64K "19 --zstd=strat=9" # btultra2
$ECHO "\n===> Pass-Through mode "
tests/regression/.gitignore
+3
View File
@@ -0,0 +1,3 @@
# regression test artifacts
data-cache
test
tests/regression/config.c
+6 -4
View File
@@ -77,10 +77,11 @@ int config_skip_data(config_t const* config, data_t const* data) {
return config->use_dictionary && !data_has_dict(data);
}
int config_get_level(config_t const* config) {
int config_get_level(config_t const* config)
{
param_values_t const params = config->param_values;
size_t i;
for (size_t i = 0; i < params.size; ++i) {
for (i = 0; i < params.size; ++i) {
if (params.data[i].param == ZSTD_c_compressionLevel)
return (int)params.data[i].value;
}
@@ -90,7 +91,8 @@ int config_get_level(config_t const* config) {
ZSTD_parameters config_get_zstd_params(
config_t const* config,
uint64_t srcSize,
size_t dictSize) {
size_t dictSize)
{
ZSTD_parameters zparams = {};
param_values_t const params = config->param_values;
int level = config_get_level(config);
@@ -130,7 +132,7 @@ ZSTD_parameters config_get_zstd_params(
case ZSTD_c_targetLength:
zparams.cParams.targetLength = value;
break;
case ZSTD_c_compressionStrategy:
case ZSTD_c_strategy:
zparams.cParams.strategy = (ZSTD_strategy)value;
break;
default:
tests/regression/results.csv
+125 -125
View File
@@ -2,17 +2,17 @@ Data, Config, Method,
silesia.tar, level -5, compress simple, 7160438
silesia.tar, level -3, compress simple, 6789024
silesia.tar, level -1, compress simple, 6195462
silesia.tar, level 0, compress simple, 4875071
silesia.tar, level 0, compress simple, 4875008
silesia.tar, level 1, compress simple, 5339697
silesia.tar, level 3, compress simple, 4875071
silesia.tar, level 4, compress simple, 4813104
silesia.tar, level 5, compress simple, 4726961
silesia.tar, level 6, compress simple, 4654401
silesia.tar, level 7, compress simple, 4591933
silesia.tar, level 3, compress simple, 4875008
silesia.tar, level 4, compress simple, 4813507
silesia.tar, level 5, compress simple, 4722235
silesia.tar, level 6, compress simple, 4672194
silesia.tar, level 7, compress simple, 4606658
silesia.tar, level 9, compress simple, 4554098
silesia.tar, level 13, compress simple, 4503496
silesia.tar, level 16, compress simple, 4387233
silesia.tar, level 19, compress simple, 4283123
silesia.tar, level 13, compress simple, 4491702
silesia.tar, level 16, compress simple, 4381277
silesia.tar, level 19, compress simple, 4281581
silesia, level -5, compress cctx, 7152294
silesia, level -3, compress cctx, 6789969
silesia, level -1, compress cctx, 6191548
@@ -20,13 +20,13 @@ silesia, level 0, compress cctx,
silesia, level 1, compress cctx, 5318036
silesia, level 3, compress cctx, 4862377
silesia, level 4, compress cctx, 4800629
silesia, level 5, compress cctx, 4715005
silesia, level 6, compress cctx, 4644055
silesia, level 7, compress cctx, 4581559
silesia, level 5, compress cctx, 4710178
silesia, level 6, compress cctx, 4659996
silesia, level 7, compress cctx, 4596234
silesia, level 9, compress cctx, 4543862
silesia, level 13, compress cctx, 4493931
silesia, level 16, compress cctx, 4381885
silesia, level 19, compress cctx, 4296899
silesia, level 13, compress cctx, 4482073
silesia, level 16, compress cctx, 4377391
silesia, level 19, compress cctx, 4294841
github, level -5, compress cctx, 232744
github, level -5 with dict, compress cctx, 45704
github, level -3, compress cctx, 220611
@@ -49,11 +49,11 @@ github, level 7, compress cctx,
github, level 7 with dict, compress cctx, 38741
github, level 9, compress cctx, 135108
github, level 9 with dict, compress cctx, 39335
github, level 13, compress cctx, 133741
github, level 13 with dict, compress cctx, 39670
github, level 16, compress cctx, 133741
github, level 16 with dict, compress cctx, 37928
github, level 19, compress cctx, 133717
github, level 13, compress cctx, 133717
github, level 13 with dict, compress cctx, 39923
github, level 16, compress cctx, 134844
github, level 16 with dict, compress cctx, 37568
github, level 19, compress cctx, 134675
github, level 19 with dict, compress cctx, 37567
silesia, level -5, zstdcli, 7152342
silesia, level -3, zstdcli, 6790021
@@ -62,28 +62,28 @@ silesia, level 0, zstdcli,
silesia, level 1, zstdcli, 5318084
silesia, level 3, zstdcli, 4862425
silesia, level 4, zstdcli, 4800677
silesia, level 5, zstdcli, 4715053
silesia, level 6, zstdcli, 4644103
silesia, level 7, zstdcli, 4581607
silesia, level 5, zstdcli, 4710226
silesia, level 6, zstdcli, 4660044
silesia, level 7, zstdcli, 4596282
silesia, level 9, zstdcli, 4543910
silesia, level 13, zstdcli, 4493979
silesia, level 16, zstdcli, 4381933
silesia, level 19, zstdcli, 4296947
silesia, level 13, zstdcli, 4482121
silesia, level 16, zstdcli, 4377439
silesia, level 19, zstdcli, 4294889
silesia.tar, level -5, zstdcli, 7159586
silesia.tar, level -3, zstdcli, 6791018
silesia.tar, level -1, zstdcli, 6196283
silesia.tar, level 0, zstdcli, 4876730
silesia.tar, level 0, zstdcli, 4875213
silesia.tar, level 1, zstdcli, 5340312
silesia.tar, level 3, zstdcli, 4876730
silesia.tar, level 4, zstdcli, 4817723
silesia.tar, level 5, zstdcli, 4730389
silesia.tar, level 6, zstdcli, 4655708
silesia.tar, level 7, zstdcli, 4593407
silesia.tar, level 3, zstdcli, 4875213
silesia.tar, level 4, zstdcli, 4814545
silesia.tar, level 5, zstdcli, 4723284
silesia.tar, level 6, zstdcli, 4673591
silesia.tar, level 7, zstdcli, 4608342
silesia.tar, level 9, zstdcli, 4556135
silesia.tar, level 13, zstdcli, 4503500
silesia.tar, level 16, zstdcli, 4387237
silesia.tar, level 19, zstdcli, 4283127
silesia.tar, no source size, zstdcli, 4876726
silesia.tar, level 13, zstdcli, 4491706
silesia.tar, level 16, zstdcli, 4381281
silesia.tar, level 19, zstdcli, 4281585
silesia.tar, no source size, zstdcli, 4875209
github, level -5, zstdcli, 234744
github, level -5 with dict, zstdcli, 47528
github, level -3, zstdcli, 222611
@@ -106,11 +106,11 @@ github, level 7, zstdcli,
github, level 7 with dict, zstdcli, 40766
github, level 9, zstdcli, 137108
github, level 9 with dict, zstdcli, 41326
github, level 13, zstdcli, 135741
github, level 13 with dict, zstdcli, 41670
github, level 16, zstdcli, 135741
github, level 16 with dict, zstdcli, 39940
github, level 19, zstdcli, 135717
github, level 13, zstdcli, 135717
github, level 13 with dict, zstdcli, 41716
github, level 16, zstdcli, 136846
github, level 16 with dict, zstdcli, 39577
github, level 19, zstdcli, 136676
github, level 19 with dict, zstdcli, 39576
silesia, level -5, advanced one pass, 7152294
silesia, level -3, advanced one pass, 6789969
@@ -119,29 +119,29 @@ silesia, level 0, advanced one pass,
silesia, level 1, advanced one pass, 5318036
silesia, level 3, advanced one pass, 4862377
silesia, level 4, advanced one pass, 4800629
silesia, level 5, advanced one pass, 4715005
silesia, level 6, advanced one pass, 4644055
silesia, level 7, advanced one pass, 4581559
silesia, level 5, advanced one pass, 4710178
silesia, level 6, advanced one pass, 4659996
silesia, level 7, advanced one pass, 4596234
silesia, level 9, advanced one pass, 4543862
silesia, level 13, advanced one pass, 4493931
silesia, level 16, advanced one pass, 4381885
silesia, level 19, advanced one pass, 4296899
silesia, level 13, advanced one pass, 4482073
silesia, level 16, advanced one pass, 4377391
silesia, level 19, advanced one pass, 4294841
silesia, no source size, advanced one pass, 4862377
silesia.tar, level -5, advanced one pass, 7160438
silesia.tar, level -3, advanced one pass, 6789024
silesia.tar, level -1, advanced one pass, 6195462
silesia.tar, level 0, advanced one pass, 4875071
silesia.tar, level 0, advanced one pass, 4875008
silesia.tar, level 1, advanced one pass, 5339697
silesia.tar, level 3, advanced one pass, 4875071
silesia.tar, level 4, advanced one pass, 4813104
silesia.tar, level 5, advanced one pass, 4726961
silesia.tar, level 6, advanced one pass, 4654401
silesia.tar, level 7, advanced one pass, 4591933
silesia.tar, level 3, advanced one pass, 4875008
silesia.tar, level 4, advanced one pass, 4813507
silesia.tar, level 5, advanced one pass, 4722235
silesia.tar, level 6, advanced one pass, 4672194
silesia.tar, level 7, advanced one pass, 4606658
silesia.tar, level 9, advanced one pass, 4554098
silesia.tar, level 13, advanced one pass, 4503496
silesia.tar, level 16, advanced one pass, 4387233
silesia.tar, level 19, advanced one pass, 4283123
silesia.tar, no source size, advanced one pass, 4875071
silesia.tar, level 13, advanced one pass, 4491702
silesia.tar, level 16, advanced one pass, 4381277
silesia.tar, level 19, advanced one pass, 4281581
silesia.tar, no source size, advanced one pass, 4875008
github, level -5, advanced one pass, 232744
github, level -5 with dict, advanced one pass, 45528
github, level -3, advanced one pass, 220611
@@ -164,11 +164,11 @@ github, level 7, advanced one pass,
github, level 7 with dict, advanced one pass, 38766
github, level 9, advanced one pass, 135108
github, level 9 with dict, advanced one pass, 39326
github, level 13, advanced one pass, 133741
github, level 13 with dict, advanced one pass, 39670
github, level 16, advanced one pass, 133741
github, level 16 with dict, advanced one pass, 37940
github, level 19, advanced one pass, 133717
github, level 13, advanced one pass, 133717
github, level 13 with dict, advanced one pass, 39716
github, level 16, advanced one pass, 134844
github, level 16 with dict, advanced one pass, 37577
github, level 19, advanced one pass, 134675
github, level 19 with dict, advanced one pass, 37576
github, no source size, advanced one pass, 136397
silesia, level -5, advanced one pass small out, 7152294
@@ -178,29 +178,29 @@ silesia, level 0, advanced one pass smal
silesia, level 1, advanced one pass small out, 5318036
silesia, level 3, advanced one pass small out, 4862377
silesia, level 4, advanced one pass small out, 4800629
silesia, level 5, advanced one pass small out, 4715005
silesia, level 6, advanced one pass small out, 4644055
silesia, level 7, advanced one pass small out, 4581559
silesia, level 5, advanced one pass small out, 4710178
silesia, level 6, advanced one pass small out, 4659996
silesia, level 7, advanced one pass small out, 4596234
silesia, level 9, advanced one pass small out, 4543862
silesia, level 13, advanced one pass small out, 4493931
silesia, level 16, advanced one pass small out, 4381885
silesia, level 19, advanced one pass small out, 4296899
silesia, level 13, advanced one pass small out, 4482073
silesia, level 16, advanced one pass small out, 4377391
silesia, level 19, advanced one pass small out, 4294841
silesia, no source size, advanced one pass small out, 4862377
silesia.tar, level -5, advanced one pass small out, 7160438
silesia.tar, level -3, advanced one pass small out, 6789024
silesia.tar, level -1, advanced one pass small out, 6195462
silesia.tar, level 0, advanced one pass small out, 4875071
silesia.tar, level 0, advanced one pass small out, 4875008
silesia.tar, level 1, advanced one pass small out, 5339697
silesia.tar, level 3, advanced one pass small out, 4875071
silesia.tar, level 4, advanced one pass small out, 4813104
silesia.tar, level 5, advanced one pass small out, 4726961
silesia.tar, level 6, advanced one pass small out, 4654401
silesia.tar, level 7, advanced one pass small out, 4591933
silesia.tar, level 3, advanced one pass small out, 4875008
silesia.tar, level 4, advanced one pass small out, 4813507
silesia.tar, level 5, advanced one pass small out, 4722235
silesia.tar, level 6, advanced one pass small out, 4672194
silesia.tar, level 7, advanced one pass small out, 4606658
silesia.tar, level 9, advanced one pass small out, 4554098
silesia.tar, level 13, advanced one pass small out, 4503496
silesia.tar, level 16, advanced one pass small out, 4387233
silesia.tar, level 19, advanced one pass small out, 4283123
silesia.tar, no source size, advanced one pass small out, 4875071
silesia.tar, level 13, advanced one pass small out, 4491702
silesia.tar, level 16, advanced one pass small out, 4381277
silesia.tar, level 19, advanced one pass small out, 4281581
silesia.tar, no source size, advanced one pass small out, 4875008
github, level -5, advanced one pass small out, 232744
github, level -5 with dict, advanced one pass small out, 45528
github, level -3, advanced one pass small out, 220611
@@ -223,11 +223,11 @@ github, level 7, advanced one pass smal
github, level 7 with dict, advanced one pass small out, 38766
github, level 9, advanced one pass small out, 135108
github, level 9 with dict, advanced one pass small out, 39326
github, level 13, advanced one pass small out, 133741
github, level 13 with dict, advanced one pass small out, 39670
github, level 16, advanced one pass small out, 133741
github, level 16 with dict, advanced one pass small out, 37940
github, level 19, advanced one pass small out, 133717
github, level 13, advanced one pass small out, 133717
github, level 13 with dict, advanced one pass small out, 39716
github, level 16, advanced one pass small out, 134844
github, level 16 with dict, advanced one pass small out, 37577
github, level 19, advanced one pass small out, 134675
github, level 19 with dict, advanced one pass small out, 37576
github, no source size, advanced one pass small out, 136397
silesia, level -5, advanced streaming, 7152294
@@ -237,29 +237,29 @@ silesia, level 0, advanced streaming,
silesia, level 1, advanced streaming, 5318036
silesia, level 3, advanced streaming, 4862377
silesia, level 4, advanced streaming, 4800629
silesia, level 5, advanced streaming, 4715005
silesia, level 6, advanced streaming, 4644055
silesia, level 7, advanced streaming, 4581559
silesia, level 5, advanced streaming, 4710178
silesia, level 6, advanced streaming, 4659996
silesia, level 7, advanced streaming, 4596234
silesia, level 9, advanced streaming, 4543862
silesia, level 13, advanced streaming, 4493931
silesia, level 16, advanced streaming, 4381885
silesia, level 19, advanced streaming, 4296899
silesia, level 13, advanced streaming, 4482073
silesia, level 16, advanced streaming, 4377391
silesia, level 19, advanced streaming, 4294841
silesia, no source size, advanced streaming, 4862341
silesia.tar, level -5, advanced streaming, 7160440
silesia.tar, level -3, advanced streaming, 6789026
silesia.tar, level -1, advanced streaming, 6195465
silesia.tar, level 0, advanced streaming, 4875071
silesia.tar, level 0, advanced streaming, 4875010
silesia.tar, level 1, advanced streaming, 5339701
silesia.tar, level 3, advanced streaming, 4875071
silesia.tar, level 4, advanced streaming, 4813104
silesia.tar, level 5, advanced streaming, 4726977
silesia.tar, level 6, advanced streaming, 4654404
silesia.tar, level 7, advanced streaming, 4591934
silesia.tar, level 3, advanced streaming, 4875010
silesia.tar, level 4, advanced streaming, 4813507
silesia.tar, level 5, advanced streaming, 4722240
silesia.tar, level 6, advanced streaming, 4672203
silesia.tar, level 7, advanced streaming, 4606658
silesia.tar, level 9, advanced streaming, 4554105
silesia.tar, level 13, advanced streaming, 4503496
silesia.tar, level 16, advanced streaming, 4387233
silesia.tar, level 19, advanced streaming, 4283123
silesia.tar, no source size, advanced streaming, 4875067
silesia.tar, level 13, advanced streaming, 4491703
silesia.tar, level 16, advanced streaming, 4381277
silesia.tar, level 19, advanced streaming, 4281581
silesia.tar, no source size, advanced streaming, 4875006
github, level -5, advanced streaming, 232744
github, level -5 with dict, advanced streaming, 45528
github, level -3, advanced streaming, 220611
@@ -282,11 +282,11 @@ github, level 7, advanced streaming,
github, level 7 with dict, advanced streaming, 38766
github, level 9, advanced streaming, 135108
github, level 9 with dict, advanced streaming, 39326
github, level 13, advanced streaming, 133741
github, level 13 with dict, advanced streaming, 39670
github, level 16, advanced streaming, 133741
github, level 16 with dict, advanced streaming, 37940
github, level 19, advanced streaming, 133717
github, level 13, advanced streaming, 133717
github, level 13 with dict, advanced streaming, 39716
github, level 16, advanced streaming, 134844
github, level 16 with dict, advanced streaming, 37577
github, level 19, advanced streaming, 134675
github, level 19 with dict, advanced streaming, 37576
github, no source size, advanced streaming, 136397
silesia, level -5, old streaming, 7152294
@@ -296,29 +296,29 @@ silesia, level 0, old streaming,
silesia, level 1, old streaming, 5318036
silesia, level 3, old streaming, 4862377
silesia, level 4, old streaming, 4800629
silesia, level 5, old streaming, 4715005
silesia, level 6, old streaming, 4644055
silesia, level 7, old streaming, 4581559
silesia, level 5, old streaming, 4710178
silesia, level 6, old streaming, 4659996
silesia, level 7, old streaming, 4596234
silesia, level 9, old streaming, 4543862
silesia, level 13, old streaming, 4493931
silesia, level 16, old streaming, 4381885
silesia, level 19, old streaming, 4296899
silesia, level 13, old streaming, 4482073
silesia, level 16, old streaming, 4377391
silesia, level 19, old streaming, 4294841
silesia, no source size, old streaming, 4862341
silesia.tar, level -5, old streaming, 7160440
silesia.tar, level -3, old streaming, 6789026
silesia.tar, level -1, old streaming, 6195465
silesia.tar, level 0, old streaming, 4875071
silesia.tar, level 0, old streaming, 4875010
silesia.tar, level 1, old streaming, 5339701
silesia.tar, level 3, old streaming, 4875071
silesia.tar, level 4, old streaming, 4813104
silesia.tar, level 5, old streaming, 4726977
silesia.tar, level 6, old streaming, 4654404
silesia.tar, level 7, old streaming, 4591934
silesia.tar, level 3, old streaming, 4875010
silesia.tar, level 4, old streaming, 4813507
silesia.tar, level 5, old streaming, 4722240
silesia.tar, level 6, old streaming, 4672203
silesia.tar, level 7, old streaming, 4606658
silesia.tar, level 9, old streaming, 4554105
silesia.tar, level 13, old streaming, 4503496
silesia.tar, level 16, old streaming, 4387233
silesia.tar, level 19, old streaming, 4283123
silesia.tar, no source size, old streaming, 4875067
silesia.tar, level 13, old streaming, 4491703
silesia.tar, level 16, old streaming, 4381277
silesia.tar, level 19, old streaming, 4281581
silesia.tar, no source size, old streaming, 4875006
github, level -5, old streaming, 232744
github, level -5 with dict, old streaming, 45528
github, level -3, old streaming, 220611
@@ -341,10 +341,10 @@ github, level 7, old streaming,
github, level 7 with dict, old streaming, 38766
github, level 9, old streaming, 135108
github, level 9 with dict, old streaming, 39326
github, level 13, old streaming, 133741
github, level 13 with dict, old streaming, 39670
github, level 16, old streaming, 133741
github, level 16 with dict, old streaming, 37940
github, level 19, old streaming, 133717
github, level 13, old streaming, 133717
github, level 13 with dict, old streaming, 39716
github, level 16, old streaming, 134846
github, level 16 with dict, old streaming, 37577
github, level 19, old streaming, 134676
github, level 19 with dict, old streaming, 37576
github, no source size, old streaming, 141003
1 Data Config Method Total compressed size
2 silesia.tar level -5 compress simple 7160438
3 silesia.tar level -3 compress simple 6789024
4 silesia.tar level -1 compress simple 6195462
5 silesia.tar level 0 compress simple 4875071 4875008
6 silesia.tar level 1 compress simple 5339697
7 silesia.tar level 3 compress simple 4875071 4875008
8 silesia.tar level 4 compress simple 4813104 4813507
9 silesia.tar level 5 compress simple 4726961 4722235
10 silesia.tar level 6 compress simple 4654401 4672194
11 silesia.tar level 7 compress simple 4591933 4606658
12 silesia.tar level 9 compress simple 4554098
13 silesia.tar level 13 compress simple 4503496 4491702
14 silesia.tar level 16 compress simple 4387233 4381277
15 silesia.tar level 19 compress simple 4283123 4281581
16 silesia level -5 compress cctx 7152294
17 silesia level -3 compress cctx 6789969
18 silesia level -1 compress cctx 6191548
20 silesia level 1 compress cctx 5318036
21 silesia level 3 compress cctx 4862377
22 silesia level 4 compress cctx 4800629
23 silesia level 5 compress cctx 4715005 4710178
24 silesia level 6 compress cctx 4644055 4659996
25 silesia level 7 compress cctx 4581559 4596234
26 silesia level 9 compress cctx 4543862
27 silesia level 13 compress cctx 4493931 4482073
28 silesia level 16 compress cctx 4381885 4377391
29 silesia level 19 compress cctx 4296899 4294841
30 github level -5 compress cctx 232744
31 github level -5 with dict compress cctx 45704
32 github level -3 compress cctx 220611
49 github level 7 with dict compress cctx 38741
50 github level 9 compress cctx 135108
51 github level 9 with dict compress cctx 39335
52 github level 13 compress cctx 133741 133717
53 github level 13 with dict compress cctx 39670 39923
54 github level 16 compress cctx 133741 134844
55 github level 16 with dict compress cctx 37928 37568
56 github level 19 compress cctx 133717 134675
57 github level 19 with dict compress cctx 37567
58 silesia level -5 zstdcli 7152342
59 silesia level -3 zstdcli 6790021
62 silesia level 1 zstdcli 5318084
63 silesia level 3 zstdcli 4862425
64 silesia level 4 zstdcli 4800677
65 silesia level 5 zstdcli 4715053 4710226
66 silesia level 6 zstdcli 4644103 4660044
67 silesia level 7 zstdcli 4581607 4596282
68 silesia level 9 zstdcli 4543910
69 silesia level 13 zstdcli 4493979 4482121
70 silesia level 16 zstdcli 4381933 4377439
71 silesia level 19 zstdcli 4296947 4294889
72 silesia.tar level -5 zstdcli 7159586
73 silesia.tar level -3 zstdcli 6791018
74 silesia.tar level -1 zstdcli 6196283
75 silesia.tar level 0 zstdcli 4876730 4875213
76 silesia.tar level 1 zstdcli 5340312
77 silesia.tar level 3 zstdcli 4876730 4875213
78 silesia.tar level 4 zstdcli 4817723 4814545
79 silesia.tar level 5 zstdcli 4730389 4723284
80 silesia.tar level 6 zstdcli 4655708 4673591
81 silesia.tar level 7 zstdcli 4593407 4608342
82 silesia.tar level 9 zstdcli 4556135
83 silesia.tar level 13 zstdcli 4503500 4491706
84 silesia.tar level 16 zstdcli 4387237 4381281
85 silesia.tar level 19 zstdcli 4283127 4281585
86 silesia.tar no source size zstdcli 4876726 4875209
87 github level -5 zstdcli 234744
88 github level -5 with dict zstdcli 47528
89 github level -3 zstdcli 222611
106 github level 7 with dict zstdcli 40766
107 github level 9 zstdcli 137108
108 github level 9 with dict zstdcli 41326
109 github level 13 zstdcli 135741 135717
110 github level 13 with dict zstdcli 41670 41716
111 github level 16 zstdcli 135741 136846
112 github level 16 with dict zstdcli 39940 39577
113 github level 19 zstdcli 135717 136676
114 github level 19 with dict zstdcli 39576
115 silesia level -5 advanced one pass 7152294
116 silesia level -3 advanced one pass 6789969
119 silesia level 1 advanced one pass 5318036
120 silesia level 3 advanced one pass 4862377
121 silesia level 4 advanced one pass 4800629
122 silesia level 5 advanced one pass 4715005 4710178
123 silesia level 6 advanced one pass 4644055 4659996
124 silesia level 7 advanced one pass 4581559 4596234
125 silesia level 9 advanced one pass 4543862
126 silesia level 13 advanced one pass 4493931 4482073
127 silesia level 16 advanced one pass 4381885 4377391
128 silesia level 19 advanced one pass 4296899 4294841
129 silesia no source size advanced one pass 4862377
130 silesia.tar level -5 advanced one pass 7160438
131 silesia.tar level -3 advanced one pass 6789024
132 silesia.tar level -1 advanced one pass 6195462
133 silesia.tar level 0 advanced one pass 4875071 4875008
134 silesia.tar level 1 advanced one pass 5339697
135 silesia.tar level 3 advanced one pass 4875071 4875008
136 silesia.tar level 4 advanced one pass 4813104 4813507
137 silesia.tar level 5 advanced one pass 4726961 4722235
138 silesia.tar level 6 advanced one pass 4654401 4672194
139 silesia.tar level 7 advanced one pass 4591933 4606658
140 silesia.tar level 9 advanced one pass 4554098
141 silesia.tar level 13 advanced one pass 4503496 4491702
142 silesia.tar level 16 advanced one pass 4387233 4381277
143 silesia.tar level 19 advanced one pass 4283123 4281581
144 silesia.tar no source size advanced one pass 4875071 4875008
145 github level -5 advanced one pass 232744
146 github level -5 with dict advanced one pass 45528
147 github level -3 advanced one pass 220611
164 github level 7 with dict advanced one pass 38766
165 github level 9 advanced one pass 135108
166 github level 9 with dict advanced one pass 39326
167 github level 13 advanced one pass 133741 133717
168 github level 13 with dict advanced one pass 39670 39716
169 github level 16 advanced one pass 133741 134844
170 github level 16 with dict advanced one pass 37940 37577
171 github level 19 advanced one pass 133717 134675
172 github level 19 with dict advanced one pass 37576
173 github no source size advanced one pass 136397
174 silesia level -5 advanced one pass small out 7152294
178 silesia level 1 advanced one pass small out 5318036
179 silesia level 3 advanced one pass small out 4862377
180 silesia level 4 advanced one pass small out 4800629
181 silesia level 5 advanced one pass small out 4715005 4710178
182 silesia level 6 advanced one pass small out 4644055 4659996
183 silesia level 7 advanced one pass small out 4581559 4596234
184 silesia level 9 advanced one pass small out 4543862
185 silesia level 13 advanced one pass small out 4493931 4482073
186 silesia level 16 advanced one pass small out 4381885 4377391
187 silesia level 19 advanced one pass small out 4296899 4294841
188 silesia no source size advanced one pass small out 4862377
189 silesia.tar level -5 advanced one pass small out 7160438
190 silesia.tar level -3 advanced one pass small out 6789024
191 silesia.tar level -1 advanced one pass small out 6195462
192 silesia.tar level 0 advanced one pass small out 4875071 4875008
193 silesia.tar level 1 advanced one pass small out 5339697
194 silesia.tar level 3 advanced one pass small out 4875071 4875008
195 silesia.tar level 4 advanced one pass small out 4813104 4813507
196 silesia.tar level 5 advanced one pass small out 4726961 4722235
197 silesia.tar level 6 advanced one pass small out 4654401 4672194
198 silesia.tar level 7 advanced one pass small out 4591933 4606658
199 silesia.tar level 9 advanced one pass small out 4554098
200 silesia.tar level 13 advanced one pass small out 4503496 4491702
201 silesia.tar level 16 advanced one pass small out 4387233 4381277
202 silesia.tar level 19 advanced one pass small out 4283123 4281581
203 silesia.tar no source size advanced one pass small out 4875071 4875008
204 github level -5 advanced one pass small out 232744
205 github level -5 with dict advanced one pass small out 45528
206 github level -3 advanced one pass small out 220611
223 github level 7 with dict advanced one pass small out 38766
224 github level 9 advanced one pass small out 135108
225 github level 9 with dict advanced one pass small out 39326
226 github level 13 advanced one pass small out 133741 133717
227 github level 13 with dict advanced one pass small out 39670 39716
228 github level 16 advanced one pass small out 133741 134844
229 github level 16 with dict advanced one pass small out 37940 37577
230 github level 19 advanced one pass small out 133717 134675
231 github level 19 with dict advanced one pass small out 37576
232 github no source size advanced one pass small out 136397
233 silesia level -5 advanced streaming 7152294
237 silesia level 1 advanced streaming 5318036
238 silesia level 3 advanced streaming 4862377
239 silesia level 4 advanced streaming 4800629
240 silesia level 5 advanced streaming 4715005 4710178
241 silesia level 6 advanced streaming 4644055 4659996
242 silesia level 7 advanced streaming 4581559 4596234
243 silesia level 9 advanced streaming 4543862
244 silesia level 13 advanced streaming 4493931 4482073
245 silesia level 16 advanced streaming 4381885 4377391
246 silesia level 19 advanced streaming 4296899 4294841
247 silesia no source size advanced streaming 4862341
248 silesia.tar level -5 advanced streaming 7160440
249 silesia.tar level -3 advanced streaming 6789026
250 silesia.tar level -1 advanced streaming 6195465
251 silesia.tar level 0 advanced streaming 4875071 4875010
252 silesia.tar level 1 advanced streaming 5339701
253 silesia.tar level 3 advanced streaming 4875071 4875010
254 silesia.tar level 4 advanced streaming 4813104 4813507
255 silesia.tar level 5 advanced streaming 4726977 4722240
256 silesia.tar level 6 advanced streaming 4654404 4672203
257 silesia.tar level 7 advanced streaming 4591934 4606658
258 silesia.tar level 9 advanced streaming 4554105
259 silesia.tar level 13 advanced streaming 4503496 4491703
260 silesia.tar level 16 advanced streaming 4387233 4381277
261 silesia.tar level 19 advanced streaming 4283123 4281581
262 silesia.tar no source size advanced streaming 4875067 4875006
263 github level -5 advanced streaming 232744
264 github level -5 with dict advanced streaming 45528
265 github level -3 advanced streaming 220611
282 github level 7 with dict advanced streaming 38766
283 github level 9 advanced streaming 135108
284 github level 9 with dict advanced streaming 39326
285 github level 13 advanced streaming 133741 133717
286 github level 13 with dict advanced streaming 39670 39716
287 github level 16 advanced streaming 133741 134844
288 github level 16 with dict advanced streaming 37940 37577
289 github level 19 advanced streaming 133717 134675
290 github level 19 with dict advanced streaming 37576
291 github no source size advanced streaming 136397
292 silesia level -5 old streaming 7152294
296 silesia level 1 old streaming 5318036
297 silesia level 3 old streaming 4862377
298 silesia level 4 old streaming 4800629
299 silesia level 5 old streaming 4715005 4710178
300 silesia level 6 old streaming 4644055 4659996
301 silesia level 7 old streaming 4581559 4596234
302 silesia level 9 old streaming 4543862
303 silesia level 13 old streaming 4493931 4482073
304 silesia level 16 old streaming 4381885 4377391
305 silesia level 19 old streaming 4296899 4294841
306 silesia no source size old streaming 4862341
307 silesia.tar level -5 old streaming 7160440
308 silesia.tar level -3 old streaming 6789026
309 silesia.tar level -1 old streaming 6195465
310 silesia.tar level 0 old streaming 4875071 4875010
311 silesia.tar level 1 old streaming 5339701
312 silesia.tar level 3 old streaming 4875071 4875010
313 silesia.tar level 4 old streaming 4813104 4813507
314 silesia.tar level 5 old streaming 4726977 4722240
315 silesia.tar level 6 old streaming 4654404 4672203
316 silesia.tar level 7 old streaming 4591934 4606658
317 silesia.tar level 9 old streaming 4554105
318 silesia.tar level 13 old streaming 4503496 4491703
319 silesia.tar level 16 old streaming 4387233 4381277
320 silesia.tar level 19 old streaming 4283123 4281581
321 silesia.tar no source size old streaming 4875067 4875006
322 github level -5 old streaming 232744
323 github level -5 with dict old streaming 45528
324 github level -3 old streaming 220611
341 github level 7 with dict old streaming 38766
342 github level 9 old streaming 135108
343 github level 9 with dict old streaming 39326
344 github level 13 old streaming 133741 133717
345 github level 13 with dict old streaming 39670 39716
346 github level 16 old streaming 133741 134846
347 github level 16 with dict old streaming 37940 37577
348 github level 19 old streaming 133717 134676
349 github level 19 with dict old streaming 37576
350 github no source size old streaming 141003
tests/zstreamtest.c
+2 -2
View File
@@ -230,7 +230,7 @@ static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionStrategy, &value));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
savedParams->cParams.strategy = value;
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
@@ -1548,7 +1548,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest,
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} } }