From eab692211eca2943439b7894b210a85e6931bdd9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Sep 2021 12:51:02 -0700 Subject: [PATCH 01/14] removed pretty-print of sizes in benchmark This is less appropriate for this mode : benchmark is about accuracy, it's important to read the exact values. --- lib/compress/zstd_opt.c | 3 ++- programs/benchzstd.c | 24 +++++++++--------------- programs/util.c | 7 ++++--- programs/util.h | 8 ++++++-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 301f985cb..c8d403cf5 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -987,7 +987,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, * in every price. We include the literal length to avoid negative * prices when we subtract the previous literal length. */ - opt[0].price = ZSTD_litLengthPrice(litlen, optStatePtr, optLevel); + opt[0].price = (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel); /* large match -> immediate encoding */ { U32 const maxML = matches[nbMatches-1].len; @@ -1007,6 +1007,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, } } /* set prices for first matches starting position == 0 */ + assert(opt[0].price >= 0); { U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 pos; U32 matchNb; diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 32ce60593..262efb1b1 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -181,7 +181,7 @@ BMK_initCCtx(ZSTD_CCtx* ctx, CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode)); - CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy)); + CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, (int)comprParams->strategy)); CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize)); } @@ -393,8 +393,6 @@ BMK_benchMemAdvancedNoAlloc( BMK_benchParams_t cbp, dbp; BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; - UTIL_HumanReadableSize_t hr_isize; - UTIL_HumanReadableSize_t hr_osize; cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; @@ -431,10 +429,9 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - hr_isize = UTIL_makeHumanReadableSize((U64) srcSize); - DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix); + assert(srcSize < UINT_MAX); + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u -> \r", marks[markNb], displayName, (unsigned)srcSize); while (!(compressionCompleted && decompressionCompleted)) { if (!compressionCompleted) { @@ -446,7 +443,7 @@ BMK_benchMemAdvancedNoAlloc( { BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome); cSize = cResult.sumOfReturn; - ratio = (double)srcSize / cSize; + ratio = (double)srcSize / (double)cSize; { BMK_benchResult_t newResult; newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun); benchResult.cSize = cSize; @@ -455,11 +452,10 @@ BMK_benchMemAdvancedNoAlloc( } } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - hr_osize = UTIL_makeHumanReadableSize((U64) cSize); - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s\r", + assert(cSize < UINT_MAX); + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s\r", marks[markNb], displayName, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix, + (unsigned)srcSize, (unsigned)cSize, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); } @@ -480,11 +476,9 @@ BMK_benchMemAdvancedNoAlloc( } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - hr_osize = UTIL_makeHumanReadableSize((U64) cSize); - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s, %6.1f MB/s \r", + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s, %6.1f MB/s \r", marks[markNb], displayName, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix, + (unsigned)srcSize, (unsigned)cSize, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, (double)benchResult.dSpeed / MB_UNIT); diff --git a/programs/util.c b/programs/util.c index b29177818..61cde727e 100644 --- a/programs/util.c +++ b/programs/util.c @@ -308,7 +308,8 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) return (U64)statbuf->st_size; } -UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { +UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) +{ UTIL_HumanReadableSize_t hrs; if (g_utilDisplayLevel > 3) { @@ -1121,9 +1122,9 @@ DWORD CountSetBits(ULONG_PTR bitMask) { DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; DWORD bitSetCount = 0; - ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; + ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; DWORD i; - + for (i = 0; i <= LSHIFT; ++i) { bitSetCount += ((bitMask & bitTest)?1:0); diff --git a/programs/util.h b/programs/util.h index 7632b9a0a..5e529b7eb 100644 --- a/programs/util.h +++ b/programs/util.h @@ -180,9 +180,13 @@ U64 UTIL_getFileSize(const char* infilename); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); /** - * Take a size in bytes and prepare the components to pretty-print it in a - * scaled way. The components in the returned struct should be passed in + * Take @size in bytes, + * prepare the components to pretty-print it in a scaled way. + * The components in the returned struct should be passed in * precision, value, suffix order to a "%.*f%s" format string. + * Output policy is sensible to @g_utilDisplayLevel, + * for verbose mode (@g_utilDisplayLevel >= 4), + * does not scale down. */ typedef struct { double value; From f0fc8cb3e19e4685bec012bd0bc881b2d5664899 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Sep 2021 13:44:07 -0700 Subject: [PATCH 02/14] Disable console notification by default within the library As a library, the default shouldn't be to write anything on console. `cover` and `fastcover` have a `g_displayLevel` variable to control this behavior. It's now set to 0 (no display) by default. Setting notification to a higher level should be an explicit operation by a console application. --- lib/compress/zstd_opt.c | 15 ++++++++------- lib/dictBuilder/cover.c | 4 ++-- lib/dictBuilder/fastcover.c | 8 ++++---- tests/fuzzer.c | 17 +++++++++-------- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index c8d403cf5..fe23663e6 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1008,7 +1008,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* set prices for first matches starting position == 0 */ assert(opt[0].price >= 0); - { U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 pos; U32 matchNb; for (pos = 1; pos < minMatch; pos++) { @@ -1025,7 +1025,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, opt[pos].mlen = pos; opt[pos].off = offset; opt[pos].litlen = litlen; - opt[pos].price = sequencePrice; + opt[pos].price = (int)sequencePrice; } } last_pos = pos-1; } @@ -1040,9 +1040,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* Fix current position with one literal if cheaper */ { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; int const price = opt[cur-1].price - + ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) - + ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) - - ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); assert(price < 1000000000); /* overflow check */ if (price <= opt[cur].price) { DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", @@ -1085,9 +1085,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ } + assert(opt[cur].price >= 0); { U32 const ll0 = (opt[cur].mlen != 0); U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; - U32 const previousPrice = opt[cur].price; + U32 const previousPrice = (U32)opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; @@ -1127,7 +1128,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ U32 const pos = cur + mlen; - int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); if ((pos > last_pos) || (price < opt[pos].price)) { DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 8364444d1..ec5a169c0 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -47,7 +47,7 @@ * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -735,7 +735,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( COVER_map_t activeDmers; parameters.splitPoint = 1.0; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Checks */ if (!COVER_checkParameters(parameters, dictBufferCapacity)) { DISPLAYLEVEL(1, "Cover parameters incorrect\n"); diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index ed789f92f..f4a2eed99 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -44,7 +44,7 @@ * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -549,7 +549,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, ZDICT_cover_params_t coverParams; FASTCOVER_accel_t accelParams; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Assign splitPoint and f if not provided */ parameters.splitPoint = 1.0; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; @@ -632,7 +632,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; const unsigned shrinkDict = 0; /* Local variables */ - const int displayLevel = parameters->zParams.notificationLevel; + const int displayLevel = (int)parameters->zParams.notificationLevel; unsigned iteration = 1; unsigned d; unsigned k; @@ -716,7 +716,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( data->parameters.splitPoint = splitPoint; data->parameters.steps = kSteps; data->parameters.shrinkDict = shrinkDict; - data->parameters.zParams.notificationLevel = g_displayLevel; + data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; /* Check the parameters */ if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, data->ctx->f, accel)) { diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 960050f96..92a2efa42 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1832,8 +1832,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Simple API skippable frame test */ DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); - { - U32 i; + { U32 i; unsigned readMagic; unsigned long long receivedSize; size_t skippableSize; @@ -1841,9 +1840,10 @@ static int basicUnitTests(U32 const seed, double compressibility) char* const skipBuff = (char*)malloc(skipLen); assert(skipBuff != NULL); for (i = 0; i < skipLen; i++) - skipBuff[i] = (BYTE) ((seed + i) % 256); - skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, - skipBuff, skipLen, seed % 15); + skipBuff[i] = (char) ((seed + i) % 256); + skippableSize = ZSTD_writeSkippableFrame( + compressedBuffer, compressedBufferSize, + skipBuff, skipLen, seed % 15); CHECK_Z(skippableSize); CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); @@ -1860,8 +1860,9 @@ static int basicUnitTests(U32 const seed, double compressibility) unsigned readMagic; unsigned long long receivedSize; size_t skippableSize; - skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, - CNBuffer, 0, seed % 15); + skippableSize = ZSTD_writeSkippableFrame( + compressedBuffer, compressedBufferSize, + CNBuffer, 0, seed % 15); CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); @@ -3670,7 +3671,7 @@ static int longUnitTests(U32 const seed, double compressibility) CHECK(cdict != NULL); CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, attachPref)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref)); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); CHECK_Z(cSize); From 27a8bbe26539cca301da3e013955b57e7275d82c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Sep 2021 16:07:31 -0700 Subject: [PATCH 03/14] new initializer for ll price --- lib/compress/zstd_opt.c | 47 +++++++++++----- lib/dictBuilder/zdict.c | 25 ++++++--- tests/regression/results.csv | 104 +++++++++++++++++------------------ 3 files changed, 102 insertions(+), 74 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index fe23663e6..48e1eec80 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -24,11 +24,11 @@ * Price functions for optimal parser ***************************************/ -#if 0 /* approximation at bit level */ +#if 0 /* approximation at bit level (for tests) */ # define BITCOST_ACCURACY 0 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) -#elif 0 /* fractional bit accuracy */ +# define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ # define BITCOST_ACCURACY 8 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) # define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) @@ -94,10 +94,21 @@ static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) return sum; } +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; nlitLengthSum == 0) : init statistics * take hints from dictionary if there is one - * or init from zero, using src for literals stats, or flat 1 for match symbols + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols * otherwise downscale existing stats, to be used as seed for next block. */ static void @@ -177,11 +188,16 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); } - { unsigned ll; - for (ll=0; ll<=MaxLL; ll++) - optPtr->litLengthFreq[ll] = 1; + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + //unsigned ll; for (ll=0; ll<=MaxLL; ll++) optPtr->litLengthFreq[ll] = 1; optPtr->litLengthSum = MaxLL+1; } - optPtr->litLengthSum = MaxLL+1; { unsigned ml; for (ml=0; ml<=MaxML; ml++) @@ -189,11 +205,16 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } optPtr->matchLengthSum = MaxML+1; - { unsigned of; - for (of=0; of<=MaxOff; of++) - optPtr->offCodeFreq[of] = 1; + { unsigned const baseOFCfreqs[MaxOff+1] = { + 8, 4, 3, 3, 3, 4, 4, 4, + 4, 4, 3, 3, 3, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + }; + //memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + unsigned of; for (of=0; of<=MaxOff; of++) optPtr->offCodeFreq[of] = 1; optPtr->offCodeSum = MaxOff+1; } - optPtr->offCodeSum = MaxOff+1; + } @@ -901,11 +922,11 @@ static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_ ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); } + /*-******************************* * Optimal parser *********************************/ - static U32 ZSTD_totalLen(ZSTD_optimal_t sol) { return sol.litlen + sol.mlen; diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 459cbe4d1..9edc77fe8 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -139,7 +139,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanForward64( &r, (U64)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctzll((U64)val) >> 3); + return (unsigned)(__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; @@ -150,7 +150,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanForward( &r, (U32)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); + return (unsigned)(__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; @@ -163,7 +163,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clzll(val) >> 3); + return (unsigned)(__builtin_clzll(val) >> 3); # else unsigned r; const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ @@ -178,7 +178,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); + return (unsigned)(__builtin_clz((U32)val) >> 3); # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } @@ -235,7 +235,7 @@ static dictItem ZDICT_analyzePos( U32 savings[LLIMIT] = {0}; const BYTE* b = (const BYTE*)buffer; size_t maxLength = LLIMIT; - size_t pos = suffix[start]; + size_t pos = (size_t)suffix[start]; U32 end = start; dictItem solution; @@ -369,7 +369,7 @@ static dictItem ZDICT_analyzePos( savings[i] = savings[i-1] + (lengthList[i] * (i-3)); DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", - (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength); + (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength); solution.pos = (U32)pos; solution.length = (U32)maxLength; @@ -379,7 +379,7 @@ static dictItem ZDICT_analyzePos( { U32 id; for (id=start; id= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); + int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length); table[u].savings += elt.length / 8; /* rough approx bonus */ if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; @@ -766,6 +766,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, pos += fileSizes[u]; } + if (notificationLevel >= 4) { + /* writeStats */ + DISPLAYLEVEL(4, "Offset Code Frequencies : \n"); + for (u=0; u<=offcodeMax; u++) { + DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]); + } } + /* analyze, build stats, starting with literals */ { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); if (HUF_isError(maxNbBits)) { @@ -872,7 +879,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); #else /* at this stage, we don't use the result of "most common first offset", - as the impact of statistics is not properly evaluated */ + * as the impact of statistics is not properly evaluated */ MEM_writeLE32(dstPtr+0, repStartValue[0]); MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index f120e5f2c..a552cb888 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -11,10 +11,10 @@ silesia.tar, level 6, compress silesia.tar, level 7, compress simple, 4576828 silesia.tar, level 9, compress simple, 4552584 silesia.tar, level 13, compress simple, 4502955 -silesia.tar, level 16, compress simple, 4356834 -silesia.tar, level 19, compress simple, 4264388 +silesia.tar, level 16, compress simple, 4356970 +silesia.tar, level 19, compress simple, 4264314 silesia.tar, uncompressed literals, compress simple, 4861423 -silesia.tar, uncompressed literals optimal, compress simple, 4264388 +silesia.tar, uncompressed literals optimal, compress simple, 4264314 silesia.tar, huffman literals, compress simple, 6186042 github.tar, level -5, compress simple, 46856 github.tar, level -3, compress simple, 43754 @@ -29,9 +29,9 @@ github.tar, level 7, compress github.tar, level 9, compress simple, 36767 github.tar, level 13, compress simple, 35501 github.tar, level 16, compress simple, 40255 -github.tar, level 19, compress simple, 32837 +github.tar, level 19, compress simple, 32820 github.tar, uncompressed literals, compress simple, 38441 -github.tar, uncompressed literals optimal, compress simple, 32837 +github.tar, uncompressed literals optimal, compress simple, 32820 github.tar, huffman literals, compress simple, 42490 silesia, level -5, compress cctx, 6737607 silesia, level -3, compress cctx, 6444677 @@ -45,8 +45,8 @@ silesia, level 6, compress silesia, level 7, compress cctx, 4567203 silesia, level 9, compress cctx, 4543311 silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4360251 -silesia, level 19, compress cctx, 4283236 +silesia, level 16, compress cctx, 4360072 +silesia, level 19, compress cctx, 4282792 silesia, long distance mode, compress cctx, 4849551 silesia, multithreaded, compress cctx, 4849551 silesia, multithreaded long distance mode, compress cctx, 4849551 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912199 silesia, explicit params, compress cctx, 4794480 silesia, uncompressed literals, compress cctx, 4849551 -silesia, uncompressed literals optimal, compress cctx, 4283236 +silesia, uncompressed literals optimal, compress cctx, 4282792 silesia, huffman literals, compress cctx, 6178460 silesia, multithreaded with advanced params, compress cctx, 4849551 github, level -5, compress cctx, 205285 @@ -109,8 +109,8 @@ silesia, level 6, zstdcli, silesia, level 7, zstdcli, 4567251 silesia, level 9, zstdcli, 4543359 silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4360299 -silesia, level 19, zstdcli, 4283284 +silesia, level 16, zstdcli, 4360120 +silesia, level 19, zstdcli, 4282840 silesia, long distance mode, zstdcli, 4840807 silesia, multithreaded, zstdcli, 4849599 silesia, multithreaded long distance mode, zstdcli, 4840807 @@ -119,7 +119,7 @@ silesia, small hash log, zstdcli, silesia, small chain log, zstdcli, 4912247 silesia, explicit params, zstdcli, 4795856 silesia, uncompressed literals, zstdcli, 5128030 -silesia, uncompressed literals optimal, zstdcli, 4317944 +silesia, uncompressed literals optimal, zstdcli, 4318048 silesia, huffman literals, zstdcli, 5326317 silesia, multithreaded with advanced params, zstdcli, 5128030 silesia.tar, level -5, zstdcli, 6738934 @@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli, silesia.tar, level 7, zstdcli, 4578883 silesia.tar, level 9, zstdcli, 4553498 silesia.tar, level 13, zstdcli, 4502959 -silesia.tar, level 16, zstdcli, 4356838 -silesia.tar, level 19, zstdcli, 4264392 +silesia.tar, level 16, zstdcli, 4356974 +silesia.tar, level 19, zstdcli, 4264318 silesia.tar, no source size, zstdcli, 4861507 silesia.tar, long distance mode, zstdcli, 4853225 silesia.tar, multithreaded, zstdcli, 4861511 @@ -213,7 +213,7 @@ github.tar, level 13, zstdcli, github.tar, level 13 with dict, zstdcli, 38730 github.tar, level 16, zstdcli, 40259 github.tar, level 16 with dict, zstdcli, 33643 -github.tar, level 19, zstdcli, 32841 +github.tar, level 19, zstdcli, 32824 github.tar, level 19 with dict, zstdcli, 32899 github.tar, no source size, zstdcli, 38442 github.tar, no source size with dict, zstdcli, 38004 @@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli, github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41227 github.tar, uncompressed literals, zstdcli, 41126 -github.tar, uncompressed literals optimal, zstdcli, 35392 +github.tar, uncompressed literals optimal, zstdcli, 35498 github.tar, huffman literals, zstdcli, 38781 github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 6737607 @@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass, 4503117 silesia, level 12 row 2, advanced one pass, 4505152 silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4360251 -silesia, level 19, advanced one pass, 4283236 +silesia, level 16, advanced one pass, 4360072 +silesia, level 19, advanced one pass, 4282792 silesia, no source size, advanced one pass, 4849551 silesia, long distance mode, advanced one pass, 4840738 silesia, multithreaded, advanced one pass, 4849551 @@ -259,7 +259,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass, 4912199 silesia, explicit params, advanced one pass, 4795856 silesia, uncompressed literals, advanced one pass, 5127982 -silesia, uncompressed literals optimal, advanced one pass, 4317896 +silesia, uncompressed literals optimal, advanced one pass, 4318000 silesia, huffman literals, advanced one pass, 5326269 silesia, multithreaded with advanced params, advanced one pass, 5127982 silesia.tar, level -5, advanced one pass, 6738593 @@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass, 4513603 silesia.tar, level 12 row 2, advanced one pass, 4514568 silesia.tar, level 13, advanced one pass, 4502955 -silesia.tar, level 16, advanced one pass, 4356834 -silesia.tar, level 19, advanced one pass, 4264388 +silesia.tar, level 16, advanced one pass, 4356970 +silesia.tar, level 19, advanced one pass, 4264314 silesia.tar, no source size, advanced one pass, 4861423 silesia.tar, long distance mode, advanced one pass, 4847753 silesia.tar, multithreaded, advanced one pass, 4861507 @@ -527,7 +527,7 @@ github.tar, level 16 with dict dms, advanced github.tar, level 16 with dict dds, advanced one pass, 33544 github.tar, level 16 with dict copy, advanced one pass, 33639 github.tar, level 16 with dict load, advanced one pass, 39353 -github.tar, level 19, advanced one pass, 32837 +github.tar, level 19, advanced one pass, 32820 github.tar, level 19 with dict, advanced one pass, 32895 github.tar, level 19 with dict dms, advanced one pass, 32672 github.tar, level 19 with dict dds, advanced one pass, 32672 @@ -543,7 +543,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass, 41669 github.tar, explicit params, advanced one pass, 41227 github.tar, uncompressed literals, advanced one pass, 41122 -github.tar, uncompressed literals optimal, advanced one pass, 35388 +github.tar, uncompressed literals optimal, advanced one pass, 35494 github.tar, huffman literals, advanced one pass, 38777 github.tar, multithreaded with advanced params, advanced one pass, 41122 silesia, level -5, advanced one pass small out, 6737607 @@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass small out, 4503117 silesia, level 12 row 2, advanced one pass small out, 4505152 silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4360251 -silesia, level 19, advanced one pass small out, 4283236 +silesia, level 16, advanced one pass small out, 4360072 +silesia, level 19, advanced one pass small out, 4282792 silesia, no source size, advanced one pass small out, 4849551 silesia, long distance mode, advanced one pass small out, 4840738 silesia, multithreaded, advanced one pass small out, 4849551 @@ -577,7 +577,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass small out, 4912199 silesia, explicit params, advanced one pass small out, 4795856 silesia, uncompressed literals, advanced one pass small out, 5127982 -silesia, uncompressed literals optimal, advanced one pass small out, 4317896 +silesia, uncompressed literals optimal, advanced one pass small out, 4318000 silesia, huffman literals, advanced one pass small out, 5326269 silesia, multithreaded with advanced params, advanced one pass small out, 5127982 silesia.tar, level -5, advanced one pass small out, 6738593 @@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass small out, 4513603 silesia.tar, level 12 row 2, advanced one pass small out, 4514568 silesia.tar, level 13, advanced one pass small out, 4502955 -silesia.tar, level 16, advanced one pass small out, 4356834 -silesia.tar, level 19, advanced one pass small out, 4264388 +silesia.tar, level 16, advanced one pass small out, 4356970 +silesia.tar, level 19, advanced one pass small out, 4264314 silesia.tar, no source size, advanced one pass small out, 4861423 silesia.tar, long distance mode, advanced one pass small out, 4847753 silesia.tar, multithreaded, advanced one pass small out, 4861507 @@ -845,7 +845,7 @@ github.tar, level 16 with dict dms, advanced github.tar, level 16 with dict dds, advanced one pass small out, 33544 github.tar, level 16 with dict copy, advanced one pass small out, 33639 github.tar, level 16 with dict load, advanced one pass small out, 39353 -github.tar, level 19, advanced one pass small out, 32837 +github.tar, level 19, advanced one pass small out, 32820 github.tar, level 19 with dict, advanced one pass small out, 32895 github.tar, level 19 with dict dms, advanced one pass small out, 32672 github.tar, level 19 with dict dds, advanced one pass small out, 32672 @@ -861,7 +861,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass small out, 41669 github.tar, explicit params, advanced one pass small out, 41227 github.tar, uncompressed literals, advanced one pass small out, 41122 -github.tar, uncompressed literals optimal, advanced one pass small out, 35388 +github.tar, uncompressed literals optimal, advanced one pass small out, 35494 github.tar, huffman literals, advanced one pass small out, 38777 github.tar, multithreaded with advanced params, advanced one pass small out, 41122 silesia, level -5, advanced streaming, 6882505 @@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced streaming, 4503117 silesia, level 12 row 2, advanced streaming, 4505152 silesia, level 13, advanced streaming, 4493990 -silesia, level 16, advanced streaming, 4360251 -silesia, level 19, advanced streaming, 4283236 +silesia, level 16, advanced streaming, 4360072 +silesia, level 19, advanced streaming, 4282792 silesia, no source size, advanced streaming, 4849515 silesia, long distance mode, advanced streaming, 4840738 silesia, multithreaded, advanced streaming, 4849551 @@ -895,7 +895,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced streaming, 4912199 silesia, explicit params, advanced streaming, 4795884 silesia, uncompressed literals, advanced streaming, 5127982 -silesia, uncompressed literals optimal, advanced streaming, 4317896 +silesia, uncompressed literals optimal, advanced streaming, 4318000 silesia, huffman literals, advanced streaming, 5331171 silesia, multithreaded with advanced params, advanced streaming, 5127982 silesia.tar, level -5, advanced streaming, 6982759 @@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced streaming, 4513603 silesia.tar, level 12 row 2, advanced streaming, 4514569 silesia.tar, level 13, advanced streaming, 4502955 -silesia.tar, level 16, advanced streaming, 4356834 -silesia.tar, level 19, advanced streaming, 4264388 +silesia.tar, level 16, advanced streaming, 4356970 +silesia.tar, level 19, advanced streaming, 4264314 silesia.tar, no source size, advanced streaming, 4861421 silesia.tar, long distance mode, advanced streaming, 4847753 silesia.tar, multithreaded, advanced streaming, 4861507 @@ -1163,7 +1163,7 @@ github.tar, level 16 with dict dms, advanced github.tar, level 16 with dict dds, advanced streaming, 33544 github.tar, level 16 with dict copy, advanced streaming, 33639 github.tar, level 16 with dict load, advanced streaming, 39353 -github.tar, level 19, advanced streaming, 32837 +github.tar, level 19, advanced streaming, 32820 github.tar, level 19 with dict, advanced streaming, 32895 github.tar, level 19 with dict dms, advanced streaming, 32672 github.tar, level 19 with dict dds, advanced streaming, 32672 @@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced streaming, 41669 github.tar, explicit params, advanced streaming, 41227 github.tar, uncompressed literals, advanced streaming, 41122 -github.tar, uncompressed literals optimal, advanced streaming, 35388 +github.tar, uncompressed literals optimal, advanced streaming, 35494 github.tar, huffman literals, advanced streaming, 38800 github.tar, multithreaded with advanced params, advanced streaming, 41122 silesia, level -5, old streaming, 6882505 @@ -1194,11 +1194,11 @@ silesia, level 6, old stre silesia, level 7, old streaming, 4567203 silesia, level 9, old streaming, 4543311 silesia, level 13, old streaming, 4493990 -silesia, level 16, old streaming, 4360251 -silesia, level 19, old streaming, 4283236 +silesia, level 16, old streaming, 4360072 +silesia, level 19, old streaming, 4282792 silesia, no source size, old streaming, 4849515 silesia, uncompressed literals, old streaming, 4849551 -silesia, uncompressed literals optimal, old streaming, 4283236 +silesia, uncompressed literals optimal, old streaming, 4282792 silesia, huffman literals, old streaming, 6183403 silesia.tar, level -5, old streaming, 6982759 silesia.tar, level -3, old streaming, 6641283 @@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming, 4576830 silesia.tar, level 9, old streaming, 4552590 silesia.tar, level 13, old streaming, 4502955 -silesia.tar, level 16, old streaming, 4356834 -silesia.tar, level 19, old streaming, 4264388 +silesia.tar, level 16, old streaming, 4356970 +silesia.tar, level 19, old streaming, 4264314 silesia.tar, no source size, old streaming, 4861421 silesia.tar, uncompressed literals, old streaming, 4861425 -silesia.tar, uncompressed literals optimal, old streaming, 4264388 +silesia.tar, uncompressed literals optimal, old streaming, 4264314 silesia.tar, huffman literals, old streaming, 6190795 github, level -5, old streaming, 205285 github, level -5 with dict, old streaming, 46718 @@ -1277,12 +1277,12 @@ github.tar, level 13, old stre github.tar, level 13 with dict, old streaming, 38726 github.tar, level 16, old streaming, 40255 github.tar, level 16 with dict, old streaming, 33639 -github.tar, level 19, old streaming, 32837 +github.tar, level 19, old streaming, 32820 github.tar, level 19 with dict, old streaming, 32895 github.tar, no source size, old streaming, 38438 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38441 -github.tar, uncompressed literals optimal, old streaming, 32837 +github.tar, uncompressed literals optimal, old streaming, 32820 github.tar, huffman literals, old streaming, 42465 silesia, level -5, old streaming advanced, 6882505 silesia, level -3, old streaming advanced, 6568376 @@ -1296,8 +1296,8 @@ silesia, level 6, old stre silesia, level 7, old streaming advanced, 4567203 silesia, level 9, old streaming advanced, 4543311 silesia, level 13, old streaming advanced, 4493990 -silesia, level 16, old streaming advanced, 4360251 -silesia, level 19, old streaming advanced, 4283236 +silesia, level 16, old streaming advanced, 4360072 +silesia, level 19, old streaming advanced, 4282792 silesia, no source size, old streaming advanced, 4849515 silesia, long distance mode, old streaming advanced, 4849551 silesia, multithreaded, old streaming advanced, 4849551 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912199 silesia, explicit params, old streaming advanced, 4795884 silesia, uncompressed literals, old streaming advanced, 4849551 -silesia, uncompressed literals optimal, old streaming advanced, 4283236 +silesia, uncompressed literals optimal, old streaming advanced, 4282792 silesia, huffman literals, old streaming advanced, 6183403 silesia, multithreaded with advanced params, old streaming advanced, 4849551 silesia.tar, level -5, old streaming advanced, 6982759 @@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming advanced, 4576830 silesia.tar, level 9, old streaming advanced, 4552590 silesia.tar, level 13, old streaming advanced, 4502955 -silesia.tar, level 16, old streaming advanced, 4356834 -silesia.tar, level 19, old streaming advanced, 4264388 +silesia.tar, level 16, old streaming advanced, 4356970 +silesia.tar, level 19, old streaming advanced, 4264314 silesia.tar, no source size, old streaming advanced, 4861421 silesia.tar, long distance mode, old streaming advanced, 4861425 silesia.tar, multithreaded, old streaming advanced, 4861425 @@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre silesia.tar, small chain log, old streaming advanced, 4917019 silesia.tar, explicit params, old streaming advanced, 4807403 silesia.tar, uncompressed literals, old streaming advanced, 4861425 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4264388 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4264314 silesia.tar, huffman literals, old streaming advanced, 6190795 silesia.tar, multithreaded with advanced params, old streaming advanced, 4861425 github, level -5, old streaming advanced, 216734 @@ -1403,7 +1403,7 @@ github.tar, level 13, old stre github.tar, level 13 with dict, old streaming advanced, 35807 github.tar, level 16, old streaming advanced, 40255 github.tar, level 16 with dict, old streaming advanced, 38736 -github.tar, level 19, old streaming advanced, 32837 +github.tar, level 19, old streaming advanced, 32820 github.tar, level 19 with dict, old streaming advanced, 32876 github.tar, no source size, old streaming advanced, 38438 github.tar, no source size with dict, old streaming advanced, 38015 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41227 github.tar, uncompressed literals, old streaming advanced, 38441 -github.tar, uncompressed literals optimal, old streaming advanced, 32837 +github.tar, uncompressed literals optimal, old streaming advanced, 32820 github.tar, huffman literals, old streaming advanced, 42465 github.tar, multithreaded with advanced params, old streaming advanced, 38441 github, level -5 with dict, old streaming cdict, 46718 From 23a9368c45a0c87ec1a34f8a7fbeff3807dfe967 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Sep 2021 17:41:42 -0700 Subject: [PATCH 04/14] new starting offcode table for zstd_opt --- lib/compress/zstd_opt.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 48e1eec80..784bb918a 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -196,7 +196,6 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, 1, 1, 1, 1 }; memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); - //unsigned ll; for (ll=0; ll<=MaxLL; ll++) optPtr->litLengthFreq[ll] = 1; optPtr->litLengthSum = MaxLL+1; } { unsigned ml; @@ -206,13 +205,12 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, optPtr->matchLengthSum = MaxML+1; { unsigned const baseOFCfreqs[MaxOff+1] = { - 8, 4, 3, 3, 3, 4, 4, 4, - 4, 4, 3, 3, 3, 2, 2, 2, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - }; - //memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); - unsigned of; for (of=0; of<=MaxOff; of++) optPtr->offCodeFreq[of] = 1; optPtr->offCodeSum = MaxOff+1; + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); } From 08ceda3dfc9c1e4ae7e35d210e0318a696f7f394 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 4 Sep 2021 00:52:44 -0700 Subject: [PATCH 05/14] new statistics update policy small general compression ratio improvement for btopt+ strategies/ --- lib/compress/zstd_opt.c | 50 ++++++++++++++++++++++++----------------- programs/benchzstd.c | 20 +++++++++-------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 784bb918a..a7b23e798 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -79,21 +79,6 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) } -/* ZSTD_downscaleStat() : - * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) - * return the resulting sum of elements */ -static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) -{ - U32 s, sum=0; - DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1); - assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31); - for (s=0; s> (ZSTD_FREQ_DIV+malus)); - sum += table[s]; - } - return sum; -} - static U32 sum_u32(const unsigned table[], size_t nbElts) { size_t n; @@ -104,6 +89,31 @@ static U32 sum_u32(const unsigned table[], size_t nbElts) return total; } +static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift) +{ + U32 s, sum=0; + DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift); + assert(shift < 30); + for (s=0; s> shift); + sum += table[s]; + } + return sum; +} + +/* ZSTD_scaleStats() : + * reduce all elements in table is sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor)); +} + /* ZSTD_rescaleFreqs() : * if first block (detected by optPtr->litLengthSum == 0) : init statistics * take hints from dictionary if there is one @@ -185,7 +195,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, if (compressedLiterals) { unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8); } { unsigned const baseLLfreqs[MaxLL+1] = { @@ -219,10 +229,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* new block : re-use previous statistics, scaled down */ if (compressedLiterals) - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); - optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 10); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 9); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 9); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 9); } ZSTD_setBasePrices(optPtr, optLevel); diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 262efb1b1..2366cf2d7 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -70,6 +70,8 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); } #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +#define OUTPUT(...) { fprintf(stdout, __VA_ARGS__); fflush(NULL); } +#define OUTPUTLEVEL(l, ...) if (displayLevel>=l) { OUTPUT(__VA_ARGS__); } /* ************************************* @@ -429,9 +431,9 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ + OUTPUTLEVEL(2, "\r%70s\r", ""); /* blank line */ assert(srcSize < UINT_MAX); - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u -> \r", marks[markNb], displayName, (unsigned)srcSize); + OUTPUTLEVEL(2, "%2s-%-17.17s :%10u -> \r", marks[markNb], displayName, (unsigned)srcSize); while (!(compressionCompleted && decompressionCompleted)) { if (!compressionCompleted) { @@ -453,7 +455,7 @@ BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; assert(cSize < UINT_MAX); - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s\r", + OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s\r", marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratioAccuracy, ratio, @@ -476,7 +478,7 @@ BMK_benchMemAdvancedNoAlloc( } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s, %6.1f MB/s \r", + OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s, %6.1f MB/s \r", marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratioAccuracy, ratio, @@ -537,13 +539,13 @@ BMK_benchMemAdvancedNoAlloc( double const cSpeed = (double)benchResult.cSpeed / MB_UNIT; double const dSpeed = (double)benchResult.dSpeed / MB_UNIT; if (adv->additionalParam) { - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); + OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); } else { - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); } } - DISPLAYLEVEL(2, "%2i#\n", cLevel); + OUTPUTLEVEL(2, "%2i#\n", cLevel); } /* Bench */ benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); @@ -672,7 +674,7 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS } if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ - DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", + OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10)); @@ -762,7 +764,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, } { FILE* const f = fopen(fileNamesTable[n], "rb"); if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); - DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); + OUTPUTLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); From b096a5c62632b39ec6ad6eb68bc3bafc50be8df4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Sep 2021 09:55:14 -0700 Subject: [PATCH 06/14] updated regression tests --- tests/regression/results.csv | 234 +++++++++++++++++------------------ 1 file changed, 117 insertions(+), 117 deletions(-) diff --git a/tests/regression/results.csv b/tests/regression/results.csv index a552cb888..ee02857c1 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -11,10 +11,10 @@ silesia.tar, level 6, compress silesia.tar, level 7, compress simple, 4576828 silesia.tar, level 9, compress simple, 4552584 silesia.tar, level 13, compress simple, 4502955 -silesia.tar, level 16, compress simple, 4356970 -silesia.tar, level 19, compress simple, 4264314 +silesia.tar, level 16, compress simple, 4358753 +silesia.tar, level 19, compress simple, 4263936 silesia.tar, uncompressed literals, compress simple, 4861423 -silesia.tar, uncompressed literals optimal, compress simple, 4264314 +silesia.tar, uncompressed literals optimal, compress simple, 4263936 silesia.tar, huffman literals, compress simple, 6186042 github.tar, level -5, compress simple, 46856 github.tar, level -3, compress simple, 43754 @@ -28,10 +28,10 @@ github.tar, level 6, compress github.tar, level 7, compress simple, 38073 github.tar, level 9, compress simple, 36767 github.tar, level 13, compress simple, 35501 -github.tar, level 16, compress simple, 40255 -github.tar, level 19, compress simple, 32820 +github.tar, level 16, compress simple, 40357 +github.tar, level 19, compress simple, 32506 github.tar, uncompressed literals, compress simple, 38441 -github.tar, uncompressed literals optimal, compress simple, 32820 +github.tar, uncompressed literals optimal, compress simple, 32506 github.tar, huffman literals, compress simple, 42490 silesia, level -5, compress cctx, 6737607 silesia, level -3, compress cctx, 6444677 @@ -45,8 +45,8 @@ silesia, level 6, compress silesia, level 7, compress cctx, 4567203 silesia, level 9, compress cctx, 4543311 silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4360072 -silesia, level 19, compress cctx, 4282792 +silesia, level 16, compress cctx, 4358530 +silesia, level 19, compress cctx, 4288570 silesia, long distance mode, compress cctx, 4849551 silesia, multithreaded, compress cctx, 4849551 silesia, multithreaded long distance mode, compress cctx, 4849551 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912199 silesia, explicit params, compress cctx, 4794480 silesia, uncompressed literals, compress cctx, 4849551 -silesia, uncompressed literals optimal, compress cctx, 4282792 +silesia, uncompressed literals optimal, compress cctx, 4288570 silesia, huffman literals, compress cctx, 6178460 silesia, multithreaded with advanced params, compress cctx, 4849551 github, level -5, compress cctx, 205285 @@ -109,8 +109,8 @@ silesia, level 6, zstdcli, silesia, level 7, zstdcli, 4567251 silesia, level 9, zstdcli, 4543359 silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4360120 -silesia, level 19, zstdcli, 4282840 +silesia, level 16, zstdcli, 4358578 +silesia, level 19, zstdcli, 4288618 silesia, long distance mode, zstdcli, 4840807 silesia, multithreaded, zstdcli, 4849599 silesia, multithreaded long distance mode, zstdcli, 4840807 @@ -119,7 +119,7 @@ silesia, small hash log, zstdcli, silesia, small chain log, zstdcli, 4912247 silesia, explicit params, zstdcli, 4795856 silesia, uncompressed literals, zstdcli, 5128030 -silesia, uncompressed literals optimal, zstdcli, 4318048 +silesia, uncompressed literals optimal, zstdcli, 4318014 silesia, huffman literals, zstdcli, 5326317 silesia, multithreaded with advanced params, zstdcli, 5128030 silesia.tar, level -5, zstdcli, 6738934 @@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli, silesia.tar, level 7, zstdcli, 4578883 silesia.tar, level 9, zstdcli, 4553498 silesia.tar, level 13, zstdcli, 4502959 -silesia.tar, level 16, zstdcli, 4356974 -silesia.tar, level 19, zstdcli, 4264318 +silesia.tar, level 16, zstdcli, 4358757 +silesia.tar, level 19, zstdcli, 4263940 silesia.tar, no source size, zstdcli, 4861507 silesia.tar, long distance mode, zstdcli, 4853225 silesia.tar, multithreaded, zstdcli, 4861511 @@ -145,7 +145,7 @@ silesia.tar, small hash log, zstdcli, silesia.tar, small chain log, zstdcli, 4917020 silesia.tar, explicit params, zstdcli, 4821277 silesia.tar, uncompressed literals, zstdcli, 5129559 -silesia.tar, uncompressed literals optimal, zstdcli, 4307404 +silesia.tar, uncompressed literals optimal, zstdcli, 4307883 silesia.tar, huffman literals, zstdcli, 5347610 silesia.tar, multithreaded with advanced params, zstdcli, 5129559 github, level -5, zstdcli, 207285 @@ -210,11 +210,11 @@ github.tar, level 7 with dict, zstdcli, github.tar, level 9, zstdcli, 36771 github.tar, level 9 with dict, zstdcli, 36623 github.tar, level 13, zstdcli, 35505 -github.tar, level 13 with dict, zstdcli, 38730 -github.tar, level 16, zstdcli, 40259 -github.tar, level 16 with dict, zstdcli, 33643 -github.tar, level 19, zstdcli, 32824 -github.tar, level 19 with dict, zstdcli, 32899 +github.tar, level 13 with dict, zstdcli, 37009 +github.tar, level 16, zstdcli, 40361 +github.tar, level 16 with dict, zstdcli, 33479 +github.tar, level 19, zstdcli, 32510 +github.tar, level 19 with dict, zstdcli, 32973 github.tar, no source size, zstdcli, 38442 github.tar, no source size with dict, zstdcli, 38004 github.tar, long distance mode, zstdcli, 39730 @@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli, github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41227 github.tar, uncompressed literals, zstdcli, 41126 -github.tar, uncompressed literals optimal, zstdcli, 35498 +github.tar, uncompressed literals optimal, zstdcli, 35284 github.tar, huffman literals, zstdcli, 38781 github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 6737607 @@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass, 4503117 silesia, level 12 row 2, advanced one pass, 4505152 silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4360072 -silesia, level 19, advanced one pass, 4282792 +silesia, level 16, advanced one pass, 4358530 +silesia, level 19, advanced one pass, 4288570 silesia, no source size, advanced one pass, 4849551 silesia, long distance mode, advanced one pass, 4840738 silesia, multithreaded, advanced one pass, 4849551 @@ -259,7 +259,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass, 4912199 silesia, explicit params, advanced one pass, 4795856 silesia, uncompressed literals, advanced one pass, 5127982 -silesia, uncompressed literals optimal, advanced one pass, 4318000 +silesia, uncompressed literals optimal, advanced one pass, 4317966 silesia, huffman literals, advanced one pass, 5326269 silesia, multithreaded with advanced params, advanced one pass, 5127982 silesia.tar, level -5, advanced one pass, 6738593 @@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass, 4513603 silesia.tar, level 12 row 2, advanced one pass, 4514568 silesia.tar, level 13, advanced one pass, 4502955 -silesia.tar, level 16, advanced one pass, 4356970 -silesia.tar, level 19, advanced one pass, 4264314 +silesia.tar, level 16, advanced one pass, 4358753 +silesia.tar, level 19, advanced one pass, 4263936 silesia.tar, no source size, advanced one pass, 4861423 silesia.tar, long distance mode, advanced one pass, 4847753 silesia.tar, multithreaded, advanced one pass, 4861507 @@ -293,7 +293,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass, 4917039 silesia.tar, explicit params, advanced one pass, 4807383 silesia.tar, uncompressed literals, advanced one pass, 5129458 -silesia.tar, uncompressed literals optimal, advanced one pass, 4307400 +silesia.tar, uncompressed literals optimal, advanced one pass, 4307879 silesia.tar, huffman literals, advanced one pass, 5347335 silesia.tar, multithreaded with advanced params, advanced one pass, 5129555 github, level -5, advanced one pass, 205285 @@ -516,23 +516,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced one pass, 36609 github.tar, level 12 row 2 with dict load, advanced one pass, 36460 github.tar, level 13, advanced one pass, 35501 -github.tar, level 13 with dict, advanced one pass, 38726 -github.tar, level 13 with dict dms, advanced one pass, 38903 -github.tar, level 13 with dict dds, advanced one pass, 38903 -github.tar, level 13 with dict copy, advanced one pass, 38726 +github.tar, level 13 with dict, advanced one pass, 37005 +github.tar, level 13 with dict dms, advanced one pass, 37180 +github.tar, level 13 with dict dds, advanced one pass, 37180 +github.tar, level 13 with dict copy, advanced one pass, 37005 github.tar, level 13 with dict load, advanced one pass, 36010 -github.tar, level 16, advanced one pass, 40255 -github.tar, level 16 with dict, advanced one pass, 33639 -github.tar, level 16 with dict dms, advanced one pass, 33544 -github.tar, level 16 with dict dds, advanced one pass, 33544 -github.tar, level 16 with dict copy, advanced one pass, 33639 -github.tar, level 16 with dict load, advanced one pass, 39353 -github.tar, level 19, advanced one pass, 32820 -github.tar, level 19 with dict, advanced one pass, 32895 -github.tar, level 19 with dict dms, advanced one pass, 32672 -github.tar, level 19 with dict dds, advanced one pass, 32672 -github.tar, level 19 with dict copy, advanced one pass, 32895 -github.tar, level 19 with dict load, advanced one pass, 32676 +github.tar, level 16, advanced one pass, 40357 +github.tar, level 16 with dict, advanced one pass, 33475 +github.tar, level 16 with dict dms, advanced one pass, 33357 +github.tar, level 16 with dict dds, advanced one pass, 33357 +github.tar, level 16 with dict copy, advanced one pass, 33475 +github.tar, level 16 with dict load, advanced one pass, 39064 +github.tar, level 19, advanced one pass, 32506 +github.tar, level 19 with dict, advanced one pass, 32969 +github.tar, level 19 with dict dms, advanced one pass, 32624 +github.tar, level 19 with dict dds, advanced one pass, 32624 +github.tar, level 19 with dict copy, advanced one pass, 32969 +github.tar, level 19 with dict load, advanced one pass, 32570 github.tar, no source size, advanced one pass, 38441 github.tar, no source size with dict, advanced one pass, 37995 github.tar, long distance mode, advanced one pass, 39757 @@ -543,7 +543,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass, 41669 github.tar, explicit params, advanced one pass, 41227 github.tar, uncompressed literals, advanced one pass, 41122 -github.tar, uncompressed literals optimal, advanced one pass, 35494 +github.tar, uncompressed literals optimal, advanced one pass, 35280 github.tar, huffman literals, advanced one pass, 38777 github.tar, multithreaded with advanced params, advanced one pass, 41122 silesia, level -5, advanced one pass small out, 6737607 @@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass small out, 4503117 silesia, level 12 row 2, advanced one pass small out, 4505152 silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4360072 -silesia, level 19, advanced one pass small out, 4282792 +silesia, level 16, advanced one pass small out, 4358530 +silesia, level 19, advanced one pass small out, 4288570 silesia, no source size, advanced one pass small out, 4849551 silesia, long distance mode, advanced one pass small out, 4840738 silesia, multithreaded, advanced one pass small out, 4849551 @@ -577,7 +577,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass small out, 4912199 silesia, explicit params, advanced one pass small out, 4795856 silesia, uncompressed literals, advanced one pass small out, 5127982 -silesia, uncompressed literals optimal, advanced one pass small out, 4318000 +silesia, uncompressed literals optimal, advanced one pass small out, 4317966 silesia, huffman literals, advanced one pass small out, 5326269 silesia, multithreaded with advanced params, advanced one pass small out, 5127982 silesia.tar, level -5, advanced one pass small out, 6738593 @@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass small out, 4513603 silesia.tar, level 12 row 2, advanced one pass small out, 4514568 silesia.tar, level 13, advanced one pass small out, 4502955 -silesia.tar, level 16, advanced one pass small out, 4356970 -silesia.tar, level 19, advanced one pass small out, 4264314 +silesia.tar, level 16, advanced one pass small out, 4358753 +silesia.tar, level 19, advanced one pass small out, 4263936 silesia.tar, no source size, advanced one pass small out, 4861423 silesia.tar, long distance mode, advanced one pass small out, 4847753 silesia.tar, multithreaded, advanced one pass small out, 4861507 @@ -611,7 +611,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass small out, 4917039 silesia.tar, explicit params, advanced one pass small out, 4807383 silesia.tar, uncompressed literals, advanced one pass small out, 5129458 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4307400 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4307879 silesia.tar, huffman literals, advanced one pass small out, 5347335 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555 github, level -5, advanced one pass small out, 205285 @@ -834,23 +834,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced one pass small out, 36609 github.tar, level 12 row 2 with dict load, advanced one pass small out, 36460 github.tar, level 13, advanced one pass small out, 35501 -github.tar, level 13 with dict, advanced one pass small out, 38726 -github.tar, level 13 with dict dms, advanced one pass small out, 38903 -github.tar, level 13 with dict dds, advanced one pass small out, 38903 -github.tar, level 13 with dict copy, advanced one pass small out, 38726 +github.tar, level 13 with dict, advanced one pass small out, 37005 +github.tar, level 13 with dict dms, advanced one pass small out, 37180 +github.tar, level 13 with dict dds, advanced one pass small out, 37180 +github.tar, level 13 with dict copy, advanced one pass small out, 37005 github.tar, level 13 with dict load, advanced one pass small out, 36010 -github.tar, level 16, advanced one pass small out, 40255 -github.tar, level 16 with dict, advanced one pass small out, 33639 -github.tar, level 16 with dict dms, advanced one pass small out, 33544 -github.tar, level 16 with dict dds, advanced one pass small out, 33544 -github.tar, level 16 with dict copy, advanced one pass small out, 33639 -github.tar, level 16 with dict load, advanced one pass small out, 39353 -github.tar, level 19, advanced one pass small out, 32820 -github.tar, level 19 with dict, advanced one pass small out, 32895 -github.tar, level 19 with dict dms, advanced one pass small out, 32672 -github.tar, level 19 with dict dds, advanced one pass small out, 32672 -github.tar, level 19 with dict copy, advanced one pass small out, 32895 -github.tar, level 19 with dict load, advanced one pass small out, 32676 +github.tar, level 16, advanced one pass small out, 40357 +github.tar, level 16 with dict, advanced one pass small out, 33475 +github.tar, level 16 with dict dms, advanced one pass small out, 33357 +github.tar, level 16 with dict dds, advanced one pass small out, 33357 +github.tar, level 16 with dict copy, advanced one pass small out, 33475 +github.tar, level 16 with dict load, advanced one pass small out, 39064 +github.tar, level 19, advanced one pass small out, 32506 +github.tar, level 19 with dict, advanced one pass small out, 32969 +github.tar, level 19 with dict dms, advanced one pass small out, 32624 +github.tar, level 19 with dict dds, advanced one pass small out, 32624 +github.tar, level 19 with dict copy, advanced one pass small out, 32969 +github.tar, level 19 with dict load, advanced one pass small out, 32570 github.tar, no source size, advanced one pass small out, 38441 github.tar, no source size with dict, advanced one pass small out, 37995 github.tar, long distance mode, advanced one pass small out, 39757 @@ -861,7 +861,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass small out, 41669 github.tar, explicit params, advanced one pass small out, 41227 github.tar, uncompressed literals, advanced one pass small out, 41122 -github.tar, uncompressed literals optimal, advanced one pass small out, 35494 +github.tar, uncompressed literals optimal, advanced one pass small out, 35280 github.tar, huffman literals, advanced one pass small out, 38777 github.tar, multithreaded with advanced params, advanced one pass small out, 41122 silesia, level -5, advanced streaming, 6882505 @@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced streaming, 4503117 silesia, level 12 row 2, advanced streaming, 4505152 silesia, level 13, advanced streaming, 4493990 -silesia, level 16, advanced streaming, 4360072 -silesia, level 19, advanced streaming, 4282792 +silesia, level 16, advanced streaming, 4358530 +silesia, level 19, advanced streaming, 4288570 silesia, no source size, advanced streaming, 4849515 silesia, long distance mode, advanced streaming, 4840738 silesia, multithreaded, advanced streaming, 4849551 @@ -895,7 +895,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced streaming, 4912199 silesia, explicit params, advanced streaming, 4795884 silesia, uncompressed literals, advanced streaming, 5127982 -silesia, uncompressed literals optimal, advanced streaming, 4318000 +silesia, uncompressed literals optimal, advanced streaming, 4317966 silesia, huffman literals, advanced streaming, 5331171 silesia, multithreaded with advanced params, advanced streaming, 5127982 silesia.tar, level -5, advanced streaming, 6982759 @@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced streaming, 4513603 silesia.tar, level 12 row 2, advanced streaming, 4514569 silesia.tar, level 13, advanced streaming, 4502955 -silesia.tar, level 16, advanced streaming, 4356970 -silesia.tar, level 19, advanced streaming, 4264314 +silesia.tar, level 16, advanced streaming, 4358753 +silesia.tar, level 19, advanced streaming, 4263936 silesia.tar, no source size, advanced streaming, 4861421 silesia.tar, long distance mode, advanced streaming, 4847753 silesia.tar, multithreaded, advanced streaming, 4861507 @@ -929,7 +929,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced streaming, 4917019 silesia.tar, explicit params, advanced streaming, 4807403 silesia.tar, uncompressed literals, advanced streaming, 5129461 -silesia.tar, uncompressed literals optimal, advanced streaming, 4307400 +silesia.tar, uncompressed literals optimal, advanced streaming, 4307879 silesia.tar, huffman literals, advanced streaming, 5352360 silesia.tar, multithreaded with advanced params, advanced streaming, 5129555 github, level -5, advanced streaming, 205285 @@ -1152,23 +1152,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced streaming, 36609 github.tar, level 12 row 2 with dict load, advanced streaming, 36460 github.tar, level 13, advanced streaming, 35501 -github.tar, level 13 with dict, advanced streaming, 38726 -github.tar, level 13 with dict dms, advanced streaming, 38903 -github.tar, level 13 with dict dds, advanced streaming, 38903 -github.tar, level 13 with dict copy, advanced streaming, 38726 +github.tar, level 13 with dict, advanced streaming, 37005 +github.tar, level 13 with dict dms, advanced streaming, 37180 +github.tar, level 13 with dict dds, advanced streaming, 37180 +github.tar, level 13 with dict copy, advanced streaming, 37005 github.tar, level 13 with dict load, advanced streaming, 36010 -github.tar, level 16, advanced streaming, 40255 -github.tar, level 16 with dict, advanced streaming, 33639 -github.tar, level 16 with dict dms, advanced streaming, 33544 -github.tar, level 16 with dict dds, advanced streaming, 33544 -github.tar, level 16 with dict copy, advanced streaming, 33639 -github.tar, level 16 with dict load, advanced streaming, 39353 -github.tar, level 19, advanced streaming, 32820 -github.tar, level 19 with dict, advanced streaming, 32895 -github.tar, level 19 with dict dms, advanced streaming, 32672 -github.tar, level 19 with dict dds, advanced streaming, 32672 -github.tar, level 19 with dict copy, advanced streaming, 32895 -github.tar, level 19 with dict load, advanced streaming, 32676 +github.tar, level 16, advanced streaming, 40357 +github.tar, level 16 with dict, advanced streaming, 33475 +github.tar, level 16 with dict dms, advanced streaming, 33357 +github.tar, level 16 with dict dds, advanced streaming, 33357 +github.tar, level 16 with dict copy, advanced streaming, 33475 +github.tar, level 16 with dict load, advanced streaming, 39064 +github.tar, level 19, advanced streaming, 32506 +github.tar, level 19 with dict, advanced streaming, 32969 +github.tar, level 19 with dict dms, advanced streaming, 32624 +github.tar, level 19 with dict dds, advanced streaming, 32624 +github.tar, level 19 with dict copy, advanced streaming, 32969 +github.tar, level 19 with dict load, advanced streaming, 32570 github.tar, no source size, advanced streaming, 38438 github.tar, no source size with dict, advanced streaming, 38000 github.tar, long distance mode, advanced streaming, 39757 @@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced streaming, 41669 github.tar, explicit params, advanced streaming, 41227 github.tar, uncompressed literals, advanced streaming, 41122 -github.tar, uncompressed literals optimal, advanced streaming, 35494 +github.tar, uncompressed literals optimal, advanced streaming, 35280 github.tar, huffman literals, advanced streaming, 38800 github.tar, multithreaded with advanced params, advanced streaming, 41122 silesia, level -5, old streaming, 6882505 @@ -1194,11 +1194,11 @@ silesia, level 6, old stre silesia, level 7, old streaming, 4567203 silesia, level 9, old streaming, 4543311 silesia, level 13, old streaming, 4493990 -silesia, level 16, old streaming, 4360072 -silesia, level 19, old streaming, 4282792 +silesia, level 16, old streaming, 4358530 +silesia, level 19, old streaming, 4288570 silesia, no source size, old streaming, 4849515 silesia, uncompressed literals, old streaming, 4849551 -silesia, uncompressed literals optimal, old streaming, 4282792 +silesia, uncompressed literals optimal, old streaming, 4288570 silesia, huffman literals, old streaming, 6183403 silesia.tar, level -5, old streaming, 6982759 silesia.tar, level -3, old streaming, 6641283 @@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming, 4576830 silesia.tar, level 9, old streaming, 4552590 silesia.tar, level 13, old streaming, 4502955 -silesia.tar, level 16, old streaming, 4356970 -silesia.tar, level 19, old streaming, 4264314 +silesia.tar, level 16, old streaming, 4358753 +silesia.tar, level 19, old streaming, 4263936 silesia.tar, no source size, old streaming, 4861421 silesia.tar, uncompressed literals, old streaming, 4861425 -silesia.tar, uncompressed literals optimal, old streaming, 4264314 +silesia.tar, uncompressed literals optimal, old streaming, 4263936 silesia.tar, huffman literals, old streaming, 6190795 github, level -5, old streaming, 205285 github, level -5 with dict, old streaming, 46718 @@ -1274,15 +1274,15 @@ github.tar, level 7 with dict, old stre github.tar, level 9, old streaming, 36767 github.tar, level 9 with dict, old streaming, 36457 github.tar, level 13, old streaming, 35501 -github.tar, level 13 with dict, old streaming, 38726 -github.tar, level 16, old streaming, 40255 -github.tar, level 16 with dict, old streaming, 33639 -github.tar, level 19, old streaming, 32820 -github.tar, level 19 with dict, old streaming, 32895 +github.tar, level 13 with dict, old streaming, 37005 +github.tar, level 16, old streaming, 40357 +github.tar, level 16 with dict, old streaming, 33475 +github.tar, level 19, old streaming, 32506 +github.tar, level 19 with dict, old streaming, 32969 github.tar, no source size, old streaming, 38438 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38441 -github.tar, uncompressed literals optimal, old streaming, 32820 +github.tar, uncompressed literals optimal, old streaming, 32506 github.tar, huffman literals, old streaming, 42465 silesia, level -5, old streaming advanced, 6882505 silesia, level -3, old streaming advanced, 6568376 @@ -1296,8 +1296,8 @@ silesia, level 6, old stre silesia, level 7, old streaming advanced, 4567203 silesia, level 9, old streaming advanced, 4543311 silesia, level 13, old streaming advanced, 4493990 -silesia, level 16, old streaming advanced, 4360072 -silesia, level 19, old streaming advanced, 4282792 +silesia, level 16, old streaming advanced, 4358530 +silesia, level 19, old streaming advanced, 4288570 silesia, no source size, old streaming advanced, 4849515 silesia, long distance mode, old streaming advanced, 4849551 silesia, multithreaded, old streaming advanced, 4849551 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912199 silesia, explicit params, old streaming advanced, 4795884 silesia, uncompressed literals, old streaming advanced, 4849551 -silesia, uncompressed literals optimal, old streaming advanced, 4282792 +silesia, uncompressed literals optimal, old streaming advanced, 4288570 silesia, huffman literals, old streaming advanced, 6183403 silesia, multithreaded with advanced params, old streaming advanced, 4849551 silesia.tar, level -5, old streaming advanced, 6982759 @@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming advanced, 4576830 silesia.tar, level 9, old streaming advanced, 4552590 silesia.tar, level 13, old streaming advanced, 4502955 -silesia.tar, level 16, old streaming advanced, 4356970 -silesia.tar, level 19, old streaming advanced, 4264314 +silesia.tar, level 16, old streaming advanced, 4358753 +silesia.tar, level 19, old streaming advanced, 4263936 silesia.tar, no source size, old streaming advanced, 4861421 silesia.tar, long distance mode, old streaming advanced, 4861425 silesia.tar, multithreaded, old streaming advanced, 4861425 @@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre silesia.tar, small chain log, old streaming advanced, 4917019 silesia.tar, explicit params, old streaming advanced, 4807403 silesia.tar, uncompressed literals, old streaming advanced, 4861425 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4264314 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4263936 silesia.tar, huffman literals, old streaming advanced, 6190795 silesia.tar, multithreaded with advanced params, old streaming advanced, 4861425 github, level -5, old streaming advanced, 216734 @@ -1401,10 +1401,10 @@ github.tar, level 9, old stre github.tar, level 9 with dict, old streaming advanced, 36233 github.tar, level 13, old streaming advanced, 35501 github.tar, level 13 with dict, old streaming advanced, 35807 -github.tar, level 16, old streaming advanced, 40255 -github.tar, level 16 with dict, old streaming advanced, 38736 -github.tar, level 19, old streaming advanced, 32820 -github.tar, level 19 with dict, old streaming advanced, 32876 +github.tar, level 16, old streaming advanced, 40357 +github.tar, level 16 with dict, old streaming advanced, 38730 +github.tar, level 19, old streaming advanced, 32506 +github.tar, level 19 with dict, old streaming advanced, 32900 github.tar, no source size, old streaming advanced, 38438 github.tar, no source size with dict, old streaming advanced, 38015 github.tar, long distance mode, old streaming advanced, 38441 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41227 github.tar, uncompressed literals, old streaming advanced, 38441 -github.tar, uncompressed literals optimal, old streaming advanced, 32820 +github.tar, uncompressed literals optimal, old streaming advanced, 32506 github.tar, huffman literals, old streaming advanced, 42465 github.tar, multithreaded with advanced params, old streaming advanced, 38441 github, level -5 with dict, old streaming cdict, 46718 @@ -1445,8 +1445,8 @@ github.tar, level 6 with dict, old stre github.tar, level 7 with dict, old streaming cdict, 37371 github.tar, level 9 with dict, old streaming cdict, 36352 github.tar, level 13 with dict, old streaming cdict, 36010 -github.tar, level 16 with dict, old streaming cdict, 39353 -github.tar, level 19 with dict, old streaming cdict, 32676 +github.tar, level 16 with dict, old streaming cdict, 39064 +github.tar, level 19 with dict, old streaming cdict, 32570 github.tar, no source size with dict, old streaming cdict, 38000 github, level -5 with dict, old streaming advanced cdict, 49562 github, level -3 with dict, old streaming advanced cdict, 44956 @@ -1475,6 +1475,6 @@ github.tar, level 6 with dict, old stre github.tar, level 7 with dict, old streaming advanced cdict, 37322 github.tar, level 9 with dict, old streaming advanced cdict, 36233 github.tar, level 13 with dict, old streaming advanced cdict, 35807 -github.tar, level 16 with dict, old streaming advanced cdict, 38736 -github.tar, level 19 with dict, old streaming advanced cdict, 32876 +github.tar, level 16 with dict, old streaming advanced cdict, 38730 +github.tar, level 19 with dict, old streaming advanced cdict, 32900 github.tar, no source size with dict, old streaming advanced cdict, 38015 From 42a3ed752a473d813e84ac9cf5d86589060a54ce Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Sep 2021 07:56:43 -0700 Subject: [PATCH 07/14] removed frequency booster for stat initialization of btultra2 used to be necessary to counter-balance the fixed-weight frequency update which has been recently changed for an adaptive rate (targeting stable starting frequency stats). --- lib/compress/zstd_opt.c | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index a7b23e798..f77d5eb00 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -14,7 +14,6 @@ #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ -#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ #define ZSTD_MAX_PRICE (1<<30) #define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ @@ -1253,28 +1252,7 @@ size_t ZSTD_compressBlock_btopt( } -/* used in 2-pass strategy */ -static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) -{ - U32 s, sum=0; - assert(ZSTD_FREQ_DIV+bonus >= 0); - for (s=0; slitSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); - optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); -} /* ZSTD_initStats_ultra(): * make a first compression pass, just to seed stats with more accurate starting values. @@ -1305,8 +1283,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; - /* re-inforce weight of collected statistics */ - ZSTD_upscaleStats(&ms->opt); } size_t ZSTD_compressBlock_btultra( From ef78611c269bc96cf88748cdd0e6d9e5c3ad74b7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Sep 2021 08:58:28 -0700 Subject: [PATCH 08/14] change update rate to 11/10/10/10 better for larger blocks, very small inefficiency on small block. --- lib/compress/zstd_opt.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index f77d5eb00..9f658bee4 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -228,10 +228,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* new block : re-use previous statistics, scaled down */ if (compressedLiterals) - optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 10); - optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 9); - optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 9); - optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 9); + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 11); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 10); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 10); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 10); } ZSTD_setBasePrices(optPtr, optLevel); From 7fce9a41b599eab4c5e586a984e436a38eee7121 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Sep 2021 14:05:57 -0700 Subject: [PATCH 09/14] change update rate to 12/11/11/11 better for large files, and sources with relatively "stable" entropy, like silesia.tar. slightly worse for files with rapidly changing entropy, like Calgary.tar/. Updated small files tests in fuzzer --- lib/compress/zstd_opt.c | 8 ++++---- tests/fuzzer.c | 26 ++++++++++++-------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 9f658bee4..a42bd66ef 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -228,10 +228,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* new block : re-use previous statistics, scaled down */ if (compressedLiterals) - optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 11); - optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 10); - optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 10); - optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 10); + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); } ZSTD_setBasePrices(optPtr, optLevel); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 92a2efa42..15e6d6ce0 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -42,7 +42,7 @@ #include "util.h" #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ -#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ +#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ @@ -128,7 +128,7 @@ static U32 FUZ_highbit32(U32 v32) #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; } #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) -#define CHECK(fn) { CHECK_NEWV(err, fn); } +#define CHECK(fn) { CHECK_NEWV(__err, fn); } #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } #define CHECK_OP(op, lhs, rhs) { \ @@ -1956,6 +1956,9 @@ static int basicUnitTests(U32 const seed, double compressibility) } } DISPLAYLEVEL(3, "OK \n"); + /* Note : these tests should be replaced by proper regression tests, + * but existing ones do not focus on small data + dictionary + all levels. + */ if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ size_t const flatdictSize = 22 KB; size_t const contentSize = 9 KB; @@ -1964,14 +1967,14 @@ static int basicUnitTests(U32 const seed, double compressibility) /* These upper bounds are generally within a few bytes of the compressed size */ size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, 3770, 3770, 3770, 3750, 3750, - 3742, 3670, 3670, 3660, 3660, - 3660, 3660, 3660, 3660, 3660, + 3742, 3675, 3674, 3665, 3664, + 3663, 3662, 3661, 3660, 3660, 3660, 3660, 3660 }; size_t const target_wdict_cSize[22+1] = { 2830, 2890, 2890, 2820, 2940, 2950, 2950, 2925, 2900, 2891, - 2910, 2910, 2910, 2770, 2760, - 2750, 2750, 2750, 2750, 2750, - 2750, 2750, 2750 }; + 2910, 2910, 2910, 2780, 2775, + 2765, 2760, 2755, 2754, 2753, + 2753, 2753, 2753 }; int l = 1; int const maxLevel = ZSTD_maxCLevel(); /* clevels with strategies that support rowhash on small inputs */ @@ -3472,11 +3475,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n", level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel); goto _output_error; - } - } - } - } - } + } } } } } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) @@ -3592,8 +3591,7 @@ static int longUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); - { - size_t cSizeLdm; + { size_t cSizeLdm; size_t cSizeNoLdm; ZSTD_CCtx* const cctx = ZSTD_createCCtx(); From 4f0b1b9ee5b6e1000c36e4696758ecd57cdb39b7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Sep 2021 14:37:42 -0700 Subject: [PATCH 10/14] update regression tests --- tests/regression/results.csv | 234 +++++++++++++++++------------------ 1 file changed, 117 insertions(+), 117 deletions(-) diff --git a/tests/regression/results.csv b/tests/regression/results.csv index ee02857c1..8fb176965 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -11,10 +11,10 @@ silesia.tar, level 6, compress silesia.tar, level 7, compress simple, 4576828 silesia.tar, level 9, compress simple, 4552584 silesia.tar, level 13, compress simple, 4502955 -silesia.tar, level 16, compress simple, 4358753 -silesia.tar, level 19, compress simple, 4263936 +silesia.tar, level 16, compress simple, 4360526 +silesia.tar, level 19, compress simple, 4267266 silesia.tar, uncompressed literals, compress simple, 4861423 -silesia.tar, uncompressed literals optimal, compress simple, 4263936 +silesia.tar, uncompressed literals optimal, compress simple, 4267266 silesia.tar, huffman literals, compress simple, 6186042 github.tar, level -5, compress simple, 46856 github.tar, level -3, compress simple, 43754 @@ -28,10 +28,10 @@ github.tar, level 6, compress github.tar, level 7, compress simple, 38073 github.tar, level 9, compress simple, 36767 github.tar, level 13, compress simple, 35501 -github.tar, level 16, compress simple, 40357 -github.tar, level 19, compress simple, 32506 +github.tar, level 16, compress simple, 40471 +github.tar, level 19, compress simple, 32134 github.tar, uncompressed literals, compress simple, 38441 -github.tar, uncompressed literals optimal, compress simple, 32506 +github.tar, uncompressed literals optimal, compress simple, 32134 github.tar, huffman literals, compress simple, 42490 silesia, level -5, compress cctx, 6737607 silesia, level -3, compress cctx, 6444677 @@ -45,8 +45,8 @@ silesia, level 6, compress silesia, level 7, compress cctx, 4567203 silesia, level 9, compress cctx, 4543311 silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4358530 -silesia, level 19, compress cctx, 4288570 +silesia, level 16, compress cctx, 4359864 +silesia, level 19, compress cctx, 4296880 silesia, long distance mode, compress cctx, 4849551 silesia, multithreaded, compress cctx, 4849551 silesia, multithreaded long distance mode, compress cctx, 4849551 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912199 silesia, explicit params, compress cctx, 4794480 silesia, uncompressed literals, compress cctx, 4849551 -silesia, uncompressed literals optimal, compress cctx, 4288570 +silesia, uncompressed literals optimal, compress cctx, 4296880 silesia, huffman literals, compress cctx, 6178460 silesia, multithreaded with advanced params, compress cctx, 4849551 github, level -5, compress cctx, 205285 @@ -109,8 +109,8 @@ silesia, level 6, zstdcli, silesia, level 7, zstdcli, 4567251 silesia, level 9, zstdcli, 4543359 silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4358578 -silesia, level 19, zstdcli, 4288618 +silesia, level 16, zstdcli, 4359912 +silesia, level 19, zstdcli, 4296928 silesia, long distance mode, zstdcli, 4840807 silesia, multithreaded, zstdcli, 4849599 silesia, multithreaded long distance mode, zstdcli, 4840807 @@ -119,7 +119,7 @@ silesia, small hash log, zstdcli, silesia, small chain log, zstdcli, 4912247 silesia, explicit params, zstdcli, 4795856 silesia, uncompressed literals, zstdcli, 5128030 -silesia, uncompressed literals optimal, zstdcli, 4318014 +silesia, uncompressed literals optimal, zstdcli, 4319566 silesia, huffman literals, zstdcli, 5326317 silesia, multithreaded with advanced params, zstdcli, 5128030 silesia.tar, level -5, zstdcli, 6738934 @@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli, silesia.tar, level 7, zstdcli, 4578883 silesia.tar, level 9, zstdcli, 4553498 silesia.tar, level 13, zstdcli, 4502959 -silesia.tar, level 16, zstdcli, 4358757 -silesia.tar, level 19, zstdcli, 4263940 +silesia.tar, level 16, zstdcli, 4360530 +silesia.tar, level 19, zstdcli, 4267270 silesia.tar, no source size, zstdcli, 4861507 silesia.tar, long distance mode, zstdcli, 4853225 silesia.tar, multithreaded, zstdcli, 4861511 @@ -145,7 +145,7 @@ silesia.tar, small hash log, zstdcli, silesia.tar, small chain log, zstdcli, 4917020 silesia.tar, explicit params, zstdcli, 4821277 silesia.tar, uncompressed literals, zstdcli, 5129559 -silesia.tar, uncompressed literals optimal, zstdcli, 4307883 +silesia.tar, uncompressed literals optimal, zstdcli, 4310145 silesia.tar, huffman literals, zstdcli, 5347610 silesia.tar, multithreaded with advanced params, zstdcli, 5129559 github, level -5, zstdcli, 207285 @@ -210,11 +210,11 @@ github.tar, level 7 with dict, zstdcli, github.tar, level 9, zstdcli, 36771 github.tar, level 9 with dict, zstdcli, 36623 github.tar, level 13, zstdcli, 35505 -github.tar, level 13 with dict, zstdcli, 37009 -github.tar, level 16, zstdcli, 40361 -github.tar, level 16 with dict, zstdcli, 33479 -github.tar, level 19, zstdcli, 32510 -github.tar, level 19 with dict, zstdcli, 32973 +github.tar, level 13 with dict, zstdcli, 37134 +github.tar, level 16, zstdcli, 40475 +github.tar, level 16 with dict, zstdcli, 33382 +github.tar, level 19, zstdcli, 32138 +github.tar, level 19 with dict, zstdcli, 32713 github.tar, no source size, zstdcli, 38442 github.tar, no source size with dict, zstdcli, 38004 github.tar, long distance mode, zstdcli, 39730 @@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli, github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41227 github.tar, uncompressed literals, zstdcli, 41126 -github.tar, uncompressed literals optimal, zstdcli, 35284 +github.tar, uncompressed literals optimal, zstdcli, 35401 github.tar, huffman literals, zstdcli, 38781 github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 6737607 @@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass, 4503117 silesia, level 12 row 2, advanced one pass, 4505152 silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4358530 -silesia, level 19, advanced one pass, 4288570 +silesia, level 16, advanced one pass, 4359864 +silesia, level 19, advanced one pass, 4296880 silesia, no source size, advanced one pass, 4849551 silesia, long distance mode, advanced one pass, 4840738 silesia, multithreaded, advanced one pass, 4849551 @@ -259,7 +259,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass, 4912199 silesia, explicit params, advanced one pass, 4795856 silesia, uncompressed literals, advanced one pass, 5127982 -silesia, uncompressed literals optimal, advanced one pass, 4317966 +silesia, uncompressed literals optimal, advanced one pass, 4319518 silesia, huffman literals, advanced one pass, 5326269 silesia, multithreaded with advanced params, advanced one pass, 5127982 silesia.tar, level -5, advanced one pass, 6738593 @@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass, 4513603 silesia.tar, level 12 row 2, advanced one pass, 4514568 silesia.tar, level 13, advanced one pass, 4502955 -silesia.tar, level 16, advanced one pass, 4358753 -silesia.tar, level 19, advanced one pass, 4263936 +silesia.tar, level 16, advanced one pass, 4360526 +silesia.tar, level 19, advanced one pass, 4267266 silesia.tar, no source size, advanced one pass, 4861423 silesia.tar, long distance mode, advanced one pass, 4847753 silesia.tar, multithreaded, advanced one pass, 4861507 @@ -293,7 +293,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass, 4917039 silesia.tar, explicit params, advanced one pass, 4807383 silesia.tar, uncompressed literals, advanced one pass, 5129458 -silesia.tar, uncompressed literals optimal, advanced one pass, 4307879 +silesia.tar, uncompressed literals optimal, advanced one pass, 4310141 silesia.tar, huffman literals, advanced one pass, 5347335 silesia.tar, multithreaded with advanced params, advanced one pass, 5129555 github, level -5, advanced one pass, 205285 @@ -516,23 +516,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced one pass, 36609 github.tar, level 12 row 2 with dict load, advanced one pass, 36460 github.tar, level 13, advanced one pass, 35501 -github.tar, level 13 with dict, advanced one pass, 37005 -github.tar, level 13 with dict dms, advanced one pass, 37180 -github.tar, level 13 with dict dds, advanced one pass, 37180 -github.tar, level 13 with dict copy, advanced one pass, 37005 +github.tar, level 13 with dict, advanced one pass, 37130 +github.tar, level 13 with dict dms, advanced one pass, 37267 +github.tar, level 13 with dict dds, advanced one pass, 37267 +github.tar, level 13 with dict copy, advanced one pass, 37130 github.tar, level 13 with dict load, advanced one pass, 36010 -github.tar, level 16, advanced one pass, 40357 -github.tar, level 16 with dict, advanced one pass, 33475 -github.tar, level 16 with dict dms, advanced one pass, 33357 -github.tar, level 16 with dict dds, advanced one pass, 33357 -github.tar, level 16 with dict copy, advanced one pass, 33475 -github.tar, level 16 with dict load, advanced one pass, 39064 -github.tar, level 19, advanced one pass, 32506 -github.tar, level 19 with dict, advanced one pass, 32969 -github.tar, level 19 with dict dms, advanced one pass, 32624 -github.tar, level 19 with dict dds, advanced one pass, 32624 -github.tar, level 19 with dict copy, advanced one pass, 32969 -github.tar, level 19 with dict load, advanced one pass, 32570 +github.tar, level 16, advanced one pass, 40471 +github.tar, level 16 with dict, advanced one pass, 33378 +github.tar, level 16 with dict dms, advanced one pass, 33213 +github.tar, level 16 with dict dds, advanced one pass, 33213 +github.tar, level 16 with dict copy, advanced one pass, 33378 +github.tar, level 16 with dict load, advanced one pass, 39081 +github.tar, level 19, advanced one pass, 32134 +github.tar, level 19 with dict, advanced one pass, 32709 +github.tar, level 19 with dict dms, advanced one pass, 32553 +github.tar, level 19 with dict dds, advanced one pass, 32553 +github.tar, level 19 with dict copy, advanced one pass, 32709 +github.tar, level 19 with dict load, advanced one pass, 32474 github.tar, no source size, advanced one pass, 38441 github.tar, no source size with dict, advanced one pass, 37995 github.tar, long distance mode, advanced one pass, 39757 @@ -543,7 +543,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass, 41669 github.tar, explicit params, advanced one pass, 41227 github.tar, uncompressed literals, advanced one pass, 41122 -github.tar, uncompressed literals optimal, advanced one pass, 35280 +github.tar, uncompressed literals optimal, advanced one pass, 35397 github.tar, huffman literals, advanced one pass, 38777 github.tar, multithreaded with advanced params, advanced one pass, 41122 silesia, level -5, advanced one pass small out, 6737607 @@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass small out, 4503117 silesia, level 12 row 2, advanced one pass small out, 4505152 silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4358530 -silesia, level 19, advanced one pass small out, 4288570 +silesia, level 16, advanced one pass small out, 4359864 +silesia, level 19, advanced one pass small out, 4296880 silesia, no source size, advanced one pass small out, 4849551 silesia, long distance mode, advanced one pass small out, 4840738 silesia, multithreaded, advanced one pass small out, 4849551 @@ -577,7 +577,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass small out, 4912199 silesia, explicit params, advanced one pass small out, 4795856 silesia, uncompressed literals, advanced one pass small out, 5127982 -silesia, uncompressed literals optimal, advanced one pass small out, 4317966 +silesia, uncompressed literals optimal, advanced one pass small out, 4319518 silesia, huffman literals, advanced one pass small out, 5326269 silesia, multithreaded with advanced params, advanced one pass small out, 5127982 silesia.tar, level -5, advanced one pass small out, 6738593 @@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass small out, 4513603 silesia.tar, level 12 row 2, advanced one pass small out, 4514568 silesia.tar, level 13, advanced one pass small out, 4502955 -silesia.tar, level 16, advanced one pass small out, 4358753 -silesia.tar, level 19, advanced one pass small out, 4263936 +silesia.tar, level 16, advanced one pass small out, 4360526 +silesia.tar, level 19, advanced one pass small out, 4267266 silesia.tar, no source size, advanced one pass small out, 4861423 silesia.tar, long distance mode, advanced one pass small out, 4847753 silesia.tar, multithreaded, advanced one pass small out, 4861507 @@ -611,7 +611,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass small out, 4917039 silesia.tar, explicit params, advanced one pass small out, 4807383 silesia.tar, uncompressed literals, advanced one pass small out, 5129458 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4307879 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4310141 silesia.tar, huffman literals, advanced one pass small out, 5347335 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555 github, level -5, advanced one pass small out, 205285 @@ -834,23 +834,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced one pass small out, 36609 github.tar, level 12 row 2 with dict load, advanced one pass small out, 36460 github.tar, level 13, advanced one pass small out, 35501 -github.tar, level 13 with dict, advanced one pass small out, 37005 -github.tar, level 13 with dict dms, advanced one pass small out, 37180 -github.tar, level 13 with dict dds, advanced one pass small out, 37180 -github.tar, level 13 with dict copy, advanced one pass small out, 37005 +github.tar, level 13 with dict, advanced one pass small out, 37130 +github.tar, level 13 with dict dms, advanced one pass small out, 37267 +github.tar, level 13 with dict dds, advanced one pass small out, 37267 +github.tar, level 13 with dict copy, advanced one pass small out, 37130 github.tar, level 13 with dict load, advanced one pass small out, 36010 -github.tar, level 16, advanced one pass small out, 40357 -github.tar, level 16 with dict, advanced one pass small out, 33475 -github.tar, level 16 with dict dms, advanced one pass small out, 33357 -github.tar, level 16 with dict dds, advanced one pass small out, 33357 -github.tar, level 16 with dict copy, advanced one pass small out, 33475 -github.tar, level 16 with dict load, advanced one pass small out, 39064 -github.tar, level 19, advanced one pass small out, 32506 -github.tar, level 19 with dict, advanced one pass small out, 32969 -github.tar, level 19 with dict dms, advanced one pass small out, 32624 -github.tar, level 19 with dict dds, advanced one pass small out, 32624 -github.tar, level 19 with dict copy, advanced one pass small out, 32969 -github.tar, level 19 with dict load, advanced one pass small out, 32570 +github.tar, level 16, advanced one pass small out, 40471 +github.tar, level 16 with dict, advanced one pass small out, 33378 +github.tar, level 16 with dict dms, advanced one pass small out, 33213 +github.tar, level 16 with dict dds, advanced one pass small out, 33213 +github.tar, level 16 with dict copy, advanced one pass small out, 33378 +github.tar, level 16 with dict load, advanced one pass small out, 39081 +github.tar, level 19, advanced one pass small out, 32134 +github.tar, level 19 with dict, advanced one pass small out, 32709 +github.tar, level 19 with dict dms, advanced one pass small out, 32553 +github.tar, level 19 with dict dds, advanced one pass small out, 32553 +github.tar, level 19 with dict copy, advanced one pass small out, 32709 +github.tar, level 19 with dict load, advanced one pass small out, 32474 github.tar, no source size, advanced one pass small out, 38441 github.tar, no source size with dict, advanced one pass small out, 37995 github.tar, long distance mode, advanced one pass small out, 39757 @@ -861,7 +861,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass small out, 41669 github.tar, explicit params, advanced one pass small out, 41227 github.tar, uncompressed literals, advanced one pass small out, 41122 -github.tar, uncompressed literals optimal, advanced one pass small out, 35280 +github.tar, uncompressed literals optimal, advanced one pass small out, 35397 github.tar, huffman literals, advanced one pass small out, 38777 github.tar, multithreaded with advanced params, advanced one pass small out, 41122 silesia, level -5, advanced streaming, 6882505 @@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced streaming, 4503117 silesia, level 12 row 2, advanced streaming, 4505152 silesia, level 13, advanced streaming, 4493990 -silesia, level 16, advanced streaming, 4358530 -silesia, level 19, advanced streaming, 4288570 +silesia, level 16, advanced streaming, 4359864 +silesia, level 19, advanced streaming, 4296880 silesia, no source size, advanced streaming, 4849515 silesia, long distance mode, advanced streaming, 4840738 silesia, multithreaded, advanced streaming, 4849551 @@ -895,7 +895,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced streaming, 4912199 silesia, explicit params, advanced streaming, 4795884 silesia, uncompressed literals, advanced streaming, 5127982 -silesia, uncompressed literals optimal, advanced streaming, 4317966 +silesia, uncompressed literals optimal, advanced streaming, 4319518 silesia, huffman literals, advanced streaming, 5331171 silesia, multithreaded with advanced params, advanced streaming, 5127982 silesia.tar, level -5, advanced streaming, 6982759 @@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced streaming, 4513603 silesia.tar, level 12 row 2, advanced streaming, 4514569 silesia.tar, level 13, advanced streaming, 4502955 -silesia.tar, level 16, advanced streaming, 4358753 -silesia.tar, level 19, advanced streaming, 4263936 +silesia.tar, level 16, advanced streaming, 4360526 +silesia.tar, level 19, advanced streaming, 4267266 silesia.tar, no source size, advanced streaming, 4861421 silesia.tar, long distance mode, advanced streaming, 4847753 silesia.tar, multithreaded, advanced streaming, 4861507 @@ -929,7 +929,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced streaming, 4917019 silesia.tar, explicit params, advanced streaming, 4807403 silesia.tar, uncompressed literals, advanced streaming, 5129461 -silesia.tar, uncompressed literals optimal, advanced streaming, 4307879 +silesia.tar, uncompressed literals optimal, advanced streaming, 4310141 silesia.tar, huffman literals, advanced streaming, 5352360 silesia.tar, multithreaded with advanced params, advanced streaming, 5129555 github, level -5, advanced streaming, 205285 @@ -1152,23 +1152,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced streaming, 36609 github.tar, level 12 row 2 with dict load, advanced streaming, 36460 github.tar, level 13, advanced streaming, 35501 -github.tar, level 13 with dict, advanced streaming, 37005 -github.tar, level 13 with dict dms, advanced streaming, 37180 -github.tar, level 13 with dict dds, advanced streaming, 37180 -github.tar, level 13 with dict copy, advanced streaming, 37005 +github.tar, level 13 with dict, advanced streaming, 37130 +github.tar, level 13 with dict dms, advanced streaming, 37267 +github.tar, level 13 with dict dds, advanced streaming, 37267 +github.tar, level 13 with dict copy, advanced streaming, 37130 github.tar, level 13 with dict load, advanced streaming, 36010 -github.tar, level 16, advanced streaming, 40357 -github.tar, level 16 with dict, advanced streaming, 33475 -github.tar, level 16 with dict dms, advanced streaming, 33357 -github.tar, level 16 with dict dds, advanced streaming, 33357 -github.tar, level 16 with dict copy, advanced streaming, 33475 -github.tar, level 16 with dict load, advanced streaming, 39064 -github.tar, level 19, advanced streaming, 32506 -github.tar, level 19 with dict, advanced streaming, 32969 -github.tar, level 19 with dict dms, advanced streaming, 32624 -github.tar, level 19 with dict dds, advanced streaming, 32624 -github.tar, level 19 with dict copy, advanced streaming, 32969 -github.tar, level 19 with dict load, advanced streaming, 32570 +github.tar, level 16, advanced streaming, 40471 +github.tar, level 16 with dict, advanced streaming, 33378 +github.tar, level 16 with dict dms, advanced streaming, 33213 +github.tar, level 16 with dict dds, advanced streaming, 33213 +github.tar, level 16 with dict copy, advanced streaming, 33378 +github.tar, level 16 with dict load, advanced streaming, 39081 +github.tar, level 19, advanced streaming, 32134 +github.tar, level 19 with dict, advanced streaming, 32709 +github.tar, level 19 with dict dms, advanced streaming, 32553 +github.tar, level 19 with dict dds, advanced streaming, 32553 +github.tar, level 19 with dict copy, advanced streaming, 32709 +github.tar, level 19 with dict load, advanced streaming, 32474 github.tar, no source size, advanced streaming, 38438 github.tar, no source size with dict, advanced streaming, 38000 github.tar, long distance mode, advanced streaming, 39757 @@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced streaming, 41669 github.tar, explicit params, advanced streaming, 41227 github.tar, uncompressed literals, advanced streaming, 41122 -github.tar, uncompressed literals optimal, advanced streaming, 35280 +github.tar, uncompressed literals optimal, advanced streaming, 35397 github.tar, huffman literals, advanced streaming, 38800 github.tar, multithreaded with advanced params, advanced streaming, 41122 silesia, level -5, old streaming, 6882505 @@ -1194,11 +1194,11 @@ silesia, level 6, old stre silesia, level 7, old streaming, 4567203 silesia, level 9, old streaming, 4543311 silesia, level 13, old streaming, 4493990 -silesia, level 16, old streaming, 4358530 -silesia, level 19, old streaming, 4288570 +silesia, level 16, old streaming, 4359864 +silesia, level 19, old streaming, 4296880 silesia, no source size, old streaming, 4849515 silesia, uncompressed literals, old streaming, 4849551 -silesia, uncompressed literals optimal, old streaming, 4288570 +silesia, uncompressed literals optimal, old streaming, 4296880 silesia, huffman literals, old streaming, 6183403 silesia.tar, level -5, old streaming, 6982759 silesia.tar, level -3, old streaming, 6641283 @@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming, 4576830 silesia.tar, level 9, old streaming, 4552590 silesia.tar, level 13, old streaming, 4502955 -silesia.tar, level 16, old streaming, 4358753 -silesia.tar, level 19, old streaming, 4263936 +silesia.tar, level 16, old streaming, 4360526 +silesia.tar, level 19, old streaming, 4267266 silesia.tar, no source size, old streaming, 4861421 silesia.tar, uncompressed literals, old streaming, 4861425 -silesia.tar, uncompressed literals optimal, old streaming, 4263936 +silesia.tar, uncompressed literals optimal, old streaming, 4267266 silesia.tar, huffman literals, old streaming, 6190795 github, level -5, old streaming, 205285 github, level -5 with dict, old streaming, 46718 @@ -1274,15 +1274,15 @@ github.tar, level 7 with dict, old stre github.tar, level 9, old streaming, 36767 github.tar, level 9 with dict, old streaming, 36457 github.tar, level 13, old streaming, 35501 -github.tar, level 13 with dict, old streaming, 37005 -github.tar, level 16, old streaming, 40357 -github.tar, level 16 with dict, old streaming, 33475 -github.tar, level 19, old streaming, 32506 -github.tar, level 19 with dict, old streaming, 32969 +github.tar, level 13 with dict, old streaming, 37130 +github.tar, level 16, old streaming, 40471 +github.tar, level 16 with dict, old streaming, 33378 +github.tar, level 19, old streaming, 32134 +github.tar, level 19 with dict, old streaming, 32709 github.tar, no source size, old streaming, 38438 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38441 -github.tar, uncompressed literals optimal, old streaming, 32506 +github.tar, uncompressed literals optimal, old streaming, 32134 github.tar, huffman literals, old streaming, 42465 silesia, level -5, old streaming advanced, 6882505 silesia, level -3, old streaming advanced, 6568376 @@ -1296,8 +1296,8 @@ silesia, level 6, old stre silesia, level 7, old streaming advanced, 4567203 silesia, level 9, old streaming advanced, 4543311 silesia, level 13, old streaming advanced, 4493990 -silesia, level 16, old streaming advanced, 4358530 -silesia, level 19, old streaming advanced, 4288570 +silesia, level 16, old streaming advanced, 4359864 +silesia, level 19, old streaming advanced, 4296880 silesia, no source size, old streaming advanced, 4849515 silesia, long distance mode, old streaming advanced, 4849551 silesia, multithreaded, old streaming advanced, 4849551 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912199 silesia, explicit params, old streaming advanced, 4795884 silesia, uncompressed literals, old streaming advanced, 4849551 -silesia, uncompressed literals optimal, old streaming advanced, 4288570 +silesia, uncompressed literals optimal, old streaming advanced, 4296880 silesia, huffman literals, old streaming advanced, 6183403 silesia, multithreaded with advanced params, old streaming advanced, 4849551 silesia.tar, level -5, old streaming advanced, 6982759 @@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming advanced, 4576830 silesia.tar, level 9, old streaming advanced, 4552590 silesia.tar, level 13, old streaming advanced, 4502955 -silesia.tar, level 16, old streaming advanced, 4358753 -silesia.tar, level 19, old streaming advanced, 4263936 +silesia.tar, level 16, old streaming advanced, 4360526 +silesia.tar, level 19, old streaming advanced, 4267266 silesia.tar, no source size, old streaming advanced, 4861421 silesia.tar, long distance mode, old streaming advanced, 4861425 silesia.tar, multithreaded, old streaming advanced, 4861425 @@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre silesia.tar, small chain log, old streaming advanced, 4917019 silesia.tar, explicit params, old streaming advanced, 4807403 silesia.tar, uncompressed literals, old streaming advanced, 4861425 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4263936 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4267266 silesia.tar, huffman literals, old streaming advanced, 6190795 silesia.tar, multithreaded with advanced params, old streaming advanced, 4861425 github, level -5, old streaming advanced, 216734 @@ -1401,10 +1401,10 @@ github.tar, level 9, old stre github.tar, level 9 with dict, old streaming advanced, 36233 github.tar, level 13, old streaming advanced, 35501 github.tar, level 13 with dict, old streaming advanced, 35807 -github.tar, level 16, old streaming advanced, 40357 -github.tar, level 16 with dict, old streaming advanced, 38730 -github.tar, level 19, old streaming advanced, 32506 -github.tar, level 19 with dict, old streaming advanced, 32900 +github.tar, level 16, old streaming advanced, 40471 +github.tar, level 16 with dict, old streaming advanced, 38578 +github.tar, level 19, old streaming advanced, 32134 +github.tar, level 19 with dict, old streaming advanced, 32702 github.tar, no source size, old streaming advanced, 38438 github.tar, no source size with dict, old streaming advanced, 38015 github.tar, long distance mode, old streaming advanced, 38441 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41227 github.tar, uncompressed literals, old streaming advanced, 38441 -github.tar, uncompressed literals optimal, old streaming advanced, 32506 +github.tar, uncompressed literals optimal, old streaming advanced, 32134 github.tar, huffman literals, old streaming advanced, 42465 github.tar, multithreaded with advanced params, old streaming advanced, 38441 github, level -5 with dict, old streaming cdict, 46718 @@ -1445,8 +1445,8 @@ github.tar, level 6 with dict, old stre github.tar, level 7 with dict, old streaming cdict, 37371 github.tar, level 9 with dict, old streaming cdict, 36352 github.tar, level 13 with dict, old streaming cdict, 36010 -github.tar, level 16 with dict, old streaming cdict, 39064 -github.tar, level 19 with dict, old streaming cdict, 32570 +github.tar, level 16 with dict, old streaming cdict, 39081 +github.tar, level 19 with dict, old streaming cdict, 32474 github.tar, no source size with dict, old streaming cdict, 38000 github, level -5 with dict, old streaming advanced cdict, 49562 github, level -3 with dict, old streaming advanced cdict, 44956 @@ -1475,6 +1475,6 @@ github.tar, level 6 with dict, old stre github.tar, level 7 with dict, old streaming advanced cdict, 37322 github.tar, level 9 with dict, old streaming advanced cdict, 36233 github.tar, level 13 with dict, old streaming advanced cdict, 35807 -github.tar, level 16 with dict, old streaming advanced cdict, 38730 -github.tar, level 19 with dict, old streaming advanced cdict, 32900 +github.tar, level 16 with dict, old streaming advanced cdict, 38578 +github.tar, level 19 with dict, old streaming advanced cdict, 32702 github.tar, no source size with dict, old streaming advanced cdict, 38015 From b7f46ebc234ba4ee691a2f19233179b3c90676f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Sep 2021 14:45:53 -0700 Subject: [PATCH 11/14] use ZSTD_memcpy() for better portability notably within kernel space --- lib/compress/zstd_opt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index a42bd66ef..bb43c4ed0 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -204,7 +204,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); } { unsigned ml; @@ -219,7 +219,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); } From 5449ede2e6e4f15d10b30f0908c8bd7b4182ced6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Sep 2021 15:12:28 -0700 Subject: [PATCH 12/14] make automated-benchmarking faster by employing parallel compilation of object files. --- tests/automated_benchmarking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py index 458bda451..bd995b198 100644 --- a/tests/automated_benchmarking.py +++ b/tests/automated_benchmarking.py @@ -87,7 +87,7 @@ def clone_and_build(build): git clone {github_url} zstd-{user}-{sha} && cd zstd-{user}-{sha} && {checkout_command} - make && + make -j && cd ../ """.format( user=build["user"], @@ -100,7 +100,7 @@ def clone_and_build(build): ) return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"]) else: - os.system("cd ../ && make && cd tests") + os.system("cd ../ && make -j && cd tests") return "../zstd" From 640c5b1f7740193733bb6322a783903f18150a4b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 12 Sep 2021 01:36:18 -0700 Subject: [PATCH 13/14] fix automated_benchmarking make it able to process text output sent into either stdout or stderr --- programs/benchzstd.c | 5 +---- tests/automated_benchmarking.py | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 2366cf2d7..c7675057f 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -373,10 +373,7 @@ BMK_benchMemAdvancedNoAlloc( if (adv->mode == BMK_decodeOnly) { cSizes[nbBlocks] = thisBlockSize; benchResult.cSize = thisBlockSize; - } - } - } - } + } } } } /* warming up `compressedBuffer` */ if (adv->mode == BMK_decodeOnly) { diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py index bd995b198..e6da90244 100644 --- a/tests/automated_benchmarking.py +++ b/tests/automated_benchmarking.py @@ -112,9 +112,9 @@ def parse_benchmark_output(output): def benchmark_single(executable, level, filename): return parse_benchmark_output(( subprocess.run( - [executable, "-qb{}".format(level), filename], stderr=subprocess.PIPE + [executable, "-qb{}".format(level), filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - .stderr.decode("utf-8") + .stdout.decode("utf-8") .split(" ") )) @@ -145,7 +145,7 @@ def benchmark(build, filenames, levels, iterations): def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations): cspeeds, dspeeds = [], [] for _ in range(iterations): - output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stderr=subprocess.PIPE).stderr.decode("utf-8").split(" ") + output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stdout=subprocess.PIPE).stdout.decode("utf-8").split(" ") cspeed, dspeed = parse_benchmark_output(output) cspeeds.append(cspeed) dspeeds.append(dspeed) From b6b2855b8060d20053dc81d0aa5dd1a0b822a8de Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 12 Sep 2021 10:22:35 -0700 Subject: [PATCH 14/14] updated regression tests --- tests/regression/results.csv | 262 +++++++++++++++++------------------ 1 file changed, 131 insertions(+), 131 deletions(-) diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 503e5cef2..ba48c8111 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -97,137 +97,137 @@ github, uncompressed literals, compress github, uncompressed literals optimal, compress cctx, 134064 github, huffman literals, compress cctx, 175468 github, multithreaded with advanced params, compress cctx, 141102 -silesia, level -5, zstdcli, compression error -silesia, level -3, zstdcli, compression error -silesia, level -1, zstdcli, compression error -silesia, level 0, zstdcli, compression error -silesia, level 1, zstdcli, compression error -silesia, level 3, zstdcli, compression error -silesia, level 4, zstdcli, compression error -silesia, level 5, zstdcli, compression error -silesia, level 6, zstdcli, compression error -silesia, level 7, zstdcli, compression error -silesia, level 9, zstdcli, compression error -silesia, level 13, zstdcli, compression error -silesia, level 16, zstdcli, compression error -silesia, level 19, zstdcli, compression error -silesia, long distance mode, zstdcli, compression error -silesia, multithreaded, zstdcli, compression error -silesia, multithreaded long distance mode, zstdcli, compression error -silesia, small window log, zstdcli, compression error -silesia, small hash log, zstdcli, compression error -silesia, small chain log, zstdcli, compression error -silesia, explicit params, zstdcli, compression error -silesia, uncompressed literals, zstdcli, compression error -silesia, uncompressed literals optimal, zstdcli, compression error -silesia, huffman literals, zstdcli, compression error -silesia, multithreaded with advanced params, zstdcli, compression error -silesia.tar, level -5, zstdcli, compression error -silesia.tar, level -3, zstdcli, compression error -silesia.tar, level -1, zstdcli, compression error -silesia.tar, level 0, zstdcli, compression error -silesia.tar, level 1, zstdcli, compression error -silesia.tar, level 3, zstdcli, compression error -silesia.tar, level 4, zstdcli, compression error -silesia.tar, level 5, zstdcli, compression error -silesia.tar, level 6, zstdcli, compression error -silesia.tar, level 7, zstdcli, compression error -silesia.tar, level 9, zstdcli, compression error -silesia.tar, level 13, zstdcli, compression error -silesia.tar, level 16, zstdcli, compression error -silesia.tar, level 19, zstdcli, compression error -silesia.tar, no source size, zstdcli, compression error -silesia.tar, long distance mode, zstdcli, compression error -silesia.tar, multithreaded, zstdcli, compression error -silesia.tar, multithreaded long distance mode, zstdcli, compression error -silesia.tar, small window log, zstdcli, compression error -silesia.tar, small hash log, zstdcli, compression error -silesia.tar, small chain log, zstdcli, compression error -silesia.tar, explicit params, zstdcli, compression error -silesia.tar, uncompressed literals, zstdcli, compression error -silesia.tar, uncompressed literals optimal, zstdcli, compression error -silesia.tar, huffman literals, zstdcli, compression error -silesia.tar, multithreaded with advanced params, zstdcli, compression error -github, level -5, zstdcli, compression error -github, level -5 with dict, zstdcli, compression error -github, level -3, zstdcli, compression error -github, level -3 with dict, zstdcli, compression error -github, level -1, zstdcli, compression error -github, level -1 with dict, zstdcli, compression error -github, level 0, zstdcli, compression error -github, level 0 with dict, zstdcli, compression error -github, level 1, zstdcli, compression error -github, level 1 with dict, zstdcli, compression error -github, level 3, zstdcli, compression error -github, level 3 with dict, zstdcli, compression error -github, level 4, zstdcli, compression error -github, level 4 with dict, zstdcli, compression error -github, level 5, zstdcli, compression error -github, level 5 with dict, zstdcli, compression error -github, level 6, zstdcli, compression error -github, level 6 with dict, zstdcli, compression error -github, level 7, zstdcli, compression error -github, level 7 with dict, zstdcli, compression error -github, level 9, zstdcli, compression error -github, level 9 with dict, zstdcli, compression error -github, level 13, zstdcli, compression error -github, level 13 with dict, zstdcli, compression error -github, level 16, zstdcli, compression error -github, level 16 with dict, zstdcli, compression error -github, level 19, zstdcli, compression error -github, level 19 with dict, zstdcli, compression error -github, long distance mode, zstdcli, compression error -github, multithreaded, zstdcli, compression error -github, multithreaded long distance mode, zstdcli, compression error -github, small window log, zstdcli, compression error -github, small hash log, zstdcli, compression error -github, small chain log, zstdcli, compression error -github, explicit params, zstdcli, compression error -github, uncompressed literals, zstdcli, compression error -github, uncompressed literals optimal, zstdcli, compression error -github, huffman literals, zstdcli, compression error -github, multithreaded with advanced params, zstdcli, compression error -github.tar, level -5, zstdcli, compression error -github.tar, level -5 with dict, zstdcli, compression error -github.tar, level -3, zstdcli, compression error -github.tar, level -3 with dict, zstdcli, compression error -github.tar, level -1, zstdcli, compression error -github.tar, level -1 with dict, zstdcli, compression error -github.tar, level 0, zstdcli, compression error -github.tar, level 0 with dict, zstdcli, compression error -github.tar, level 1, zstdcli, compression error -github.tar, level 1 with dict, zstdcli, compression error -github.tar, level 3, zstdcli, compression error -github.tar, level 3 with dict, zstdcli, compression error -github.tar, level 4, zstdcli, compression error -github.tar, level 4 with dict, zstdcli, compression error -github.tar, level 5, zstdcli, compression error -github.tar, level 5 with dict, zstdcli, compression error -github.tar, level 6, zstdcli, compression error -github.tar, level 6 with dict, zstdcli, compression error -github.tar, level 7, zstdcli, compression error -github.tar, level 7 with dict, zstdcli, compression error -github.tar, level 9, zstdcli, compression error -github.tar, level 9 with dict, zstdcli, compression error -github.tar, level 13, zstdcli, compression error -github.tar, level 13 with dict, zstdcli, compression error -github.tar, level 16, zstdcli, compression error -github.tar, level 16 with dict, zstdcli, compression error -github.tar, level 19, zstdcli, compression error -github.tar, level 19 with dict, zstdcli, compression error -github.tar, no source size, zstdcli, compression error -github.tar, no source size with dict, zstdcli, compression error -github.tar, long distance mode, zstdcli, compression error -github.tar, multithreaded, zstdcli, compression error -github.tar, multithreaded long distance mode, zstdcli, compression error -github.tar, small window log, zstdcli, compression error -github.tar, small hash log, zstdcli, compression error -github.tar, small chain log, zstdcli, compression error -github.tar, explicit params, zstdcli, compression error -github.tar, uncompressed literals, zstdcli, compression error -github.tar, uncompressed literals optimal, zstdcli, compression error -github.tar, huffman literals, zstdcli, compression error -github.tar, multithreaded with advanced params, zstdcli, compression error +silesia, level -5, zstdcli, 7354723 +silesia, level -3, zstdcli, 6902422 +silesia, level -1, zstdcli, 6177613 +silesia, level 0, zstdcli, 4849599 +silesia, level 1, zstdcli, 5309145 +silesia, level 3, zstdcli, 4849599 +silesia, level 4, zstdcli, 4787017 +silesia, level 5, zstdcli, 4639008 +silesia, level 6, zstdcli, 4605417 +silesia, level 7, zstdcli, 4567251 +silesia, level 9, zstdcli, 4543359 +silesia, level 13, zstdcli, 4494038 +silesia, level 16, zstdcli, 4359912 +silesia, level 19, zstdcli, 4296928 +silesia, long distance mode, zstdcli, 4840807 +silesia, multithreaded, zstdcli, 4849599 +silesia, multithreaded long distance mode, zstdcli, 4840807 +silesia, small window log, zstdcli, 7095967 +silesia, small hash log, zstdcli, 6526189 +silesia, small chain log, zstdcli, 4912247 +silesia, explicit params, zstdcli, 4795856 +silesia, uncompressed literals, zstdcli, 5128030 +silesia, uncompressed literals optimal, zstdcli, 4319566 +silesia, huffman literals, zstdcli, 5326394 +silesia, multithreaded with advanced params, zstdcli, 5128030 +silesia.tar, level -5, zstdcli, 7363866 +silesia.tar, level -3, zstdcli, 6902158 +silesia.tar, level -1, zstdcli, 6182939 +silesia.tar, level 0, zstdcli, 4861511 +silesia.tar, level 1, zstdcli, 5333184 +silesia.tar, level 3, zstdcli, 4861511 +silesia.tar, level 4, zstdcli, 4800529 +silesia.tar, level 5, zstdcli, 4651159 +silesia.tar, level 6, zstdcli, 4618402 +silesia.tar, level 7, zstdcli, 4578883 +silesia.tar, level 9, zstdcli, 4553498 +silesia.tar, level 13, zstdcli, 4502959 +silesia.tar, level 16, zstdcli, 4360530 +silesia.tar, level 19, zstdcli, 4267270 +silesia.tar, no source size, zstdcli, 4861507 +silesia.tar, long distance mode, zstdcli, 4853225 +silesia.tar, multithreaded, zstdcli, 4861511 +silesia.tar, multithreaded long distance mode, zstdcli, 4853225 +silesia.tar, small window log, zstdcli, 7101576 +silesia.tar, small hash log, zstdcli, 6529286 +silesia.tar, small chain log, zstdcli, 4917020 +silesia.tar, explicit params, zstdcli, 4821277 +silesia.tar, uncompressed literals, zstdcli, 5129559 +silesia.tar, uncompressed literals optimal, zstdcli, 4310145 +silesia.tar, huffman literals, zstdcli, 5344915 +silesia.tar, multithreaded with advanced params, zstdcli, 5129559 +github, level -5, zstdcli, 234315 +github, level -5 with dict, zstdcli, 48718 +github, level -3, zstdcli, 222760 +github, level -3 with dict, zstdcli, 47395 +github, level -1, zstdcli, 177468 +github, level -1 with dict, zstdcli, 45170 +github, level 0, zstdcli, 138335 +github, level 0 with dict, zstdcli, 43148 +github, level 1, zstdcli, 144365 +github, level 1 with dict, zstdcli, 43682 +github, level 3, zstdcli, 138335 +github, level 3 with dict, zstdcli, 43148 +github, level 4, zstdcli, 138199 +github, level 4 with dict, zstdcli, 43251 +github, level 5, zstdcli, 137121 +github, level 5 with dict, zstdcli, 40728 +github, level 6, zstdcli, 137122 +github, level 6 with dict, zstdcli, 40636 +github, level 7, zstdcli, 137122 +github, level 7 with dict, zstdcli, 40745 +github, level 9, zstdcli, 137122 +github, level 9 with dict, zstdcli, 41393 +github, level 13, zstdcli, 136064 +github, level 13 with dict, zstdcli, 41743 +github, level 16, zstdcli, 136064 +github, level 16 with dict, zstdcli, 39577 +github, level 19, zstdcli, 136064 +github, level 19 with dict, zstdcli, 39576 +github, long distance mode, zstdcli, 138335 +github, multithreaded, zstdcli, 138335 +github, multithreaded long distance mode, zstdcli, 138335 +github, small window log, zstdcli, 138335 +github, small hash log, zstdcli, 137590 +github, small chain log, zstdcli, 138341 +github, explicit params, zstdcli, 136197 +github, uncompressed literals, zstdcli, 167915 +github, uncompressed literals optimal, zstdcli, 159227 +github, huffman literals, zstdcli, 144365 +github, multithreaded with advanced params, zstdcli, 167915 +github.tar, level -5, zstdcli, 66918 +github.tar, level -5 with dict, zstdcli, 51529 +github.tar, level -3, zstdcli, 52131 +github.tar, level -3 with dict, zstdcli, 44246 +github.tar, level -1, zstdcli, 42564 +github.tar, level -1 with dict, zstdcli, 41140 +github.tar, level 0, zstdcli, 38445 +github.tar, level 0 with dict, zstdcli, 37999 +github.tar, level 1, zstdcli, 39204 +github.tar, level 1 with dict, zstdcli, 38288 +github.tar, level 3, zstdcli, 38445 +github.tar, level 3 with dict, zstdcli, 37999 +github.tar, level 4, zstdcli, 38471 +github.tar, level 4 with dict, zstdcli, 37952 +github.tar, level 5, zstdcli, 38380 +github.tar, level 5 with dict, zstdcli, 39032 +github.tar, level 6, zstdcli, 38614 +github.tar, level 6 with dict, zstdcli, 38614 +github.tar, level 7, zstdcli, 38077 +github.tar, level 7 with dict, zstdcli, 37873 +github.tar, level 9, zstdcli, 36771 +github.tar, level 9 with dict, zstdcli, 36623 +github.tar, level 13, zstdcli, 35505 +github.tar, level 13 with dict, zstdcli, 37134 +github.tar, level 16, zstdcli, 40475 +github.tar, level 16 with dict, zstdcli, 33382 +github.tar, level 19, zstdcli, 32138 +github.tar, level 19 with dict, zstdcli, 32713 +github.tar, no source size, zstdcli, 38442 +github.tar, no source size with dict, zstdcli, 38004 +github.tar, long distance mode, zstdcli, 39730 +github.tar, multithreaded, zstdcli, 38445 +github.tar, multithreaded long distance mode, zstdcli, 39730 +github.tar, small window log, zstdcli, 198544 +github.tar, small hash log, zstdcli, 129874 +github.tar, small chain log, zstdcli, 41673 +github.tar, explicit params, zstdcli, 41227 +github.tar, uncompressed literals, zstdcli, 41126 +github.tar, uncompressed literals optimal, zstdcli, 35401 +github.tar, huffman literals, zstdcli, 38857 +github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 7354675 silesia, level -3, advanced one pass, 6902374 silesia, level -1, advanced one pass, 6177565