From 5434de01e21672cdd3ac111a99a969d8c5079297 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2022 14:58:53 -0800 Subject: [PATCH 1/3] improve compression ratio of small alphabets fix #3328 In situations where the alphabet size is very small, the evaluation of literal costs from the Optimal Parser is initially incorrect. It takes some time to converge, during which compression is less efficient. This is especially important for small files, because there will not be enough data to converge, so most of the parsing is selected based on incorrect metrics. After this patch, the scenario ##3328 gets fixed, delivering the expected 29 bytes compressed size (smallest known compressed size). --- lib/compress/huf_compress.c | 16 ++++++-- lib/compress/zstd_compress.c | 11 +++--- lib/compress/zstd_compress_literals.c | 57 ++++++++++++++++++--------- lib/compress/zstd_compress_literals.h | 18 +++++---- lib/compress/zstd_opt.c | 16 +++++--- 5 files changed, 77 insertions(+), 41 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 0da96d694..007bb15e7 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -386,7 +386,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits /* renorm totalCost from 2^largestBits to 2^targetNbBits * note : totalCost is necessarily a multiple of baseCost */ - assert((totalCost & (baseCost - 1)) == 0); + assert(((U32)totalCost & (baseCost - 1)) == 0); totalCost >>= (largestBits - targetNbBits); assert(totalCost > 0); @@ -1253,7 +1253,14 @@ unsigned HUF_minTableLog(unsigned symbolCardinality) return minBitsSymbols; } -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode) +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + HUF_depth_mode depthMode) { unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); assert(srcSize > 1); /* Not supported, RLE should be used instead */ @@ -1267,12 +1274,13 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS size_t optSize = ((size_t) ~0) - 1; unsigned optLogGuess; - if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/ + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /* silently return if workspace is not large enough */ /* Search until size increases */ for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); - if (ERR_isError(maxBits)) continue; if (maxBits < optLogGuess && optLogGuess > minTableLog) break; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 759abe00a..b12cacc77 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2727,15 +2727,14 @@ ZSTD_entropyCompressSeqStore_internal( unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); size_t const litSize = (size_t)(seqStorePtr->lit - literals); - HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast; size_t const cSize = ZSTD_compressLiterals( - &prevEntropy->huf, &nextEntropy->huf, - cctxParams->cParams.strategy, - ZSTD_literalsCompressionIsDisabled(cctxParams), op, dstCapacity, literals, litSize, entropyWorkspace, entropyWkspSize, - bmi2, suspectUncompressible, depthMode); + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); assert(cSize <= dstCapacity); op += cSize; @@ -3878,12 +3877,12 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end size_t estimatedSecondHalfSize; size_t midIdx = (startIdx + endIdx)/2; + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); assert(endIdx >= startIdx); if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); return; } - DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c index 972d40e71..666e5315d 100644 --- a/lib/compress/zstd_compress_literals.c +++ b/lib/compress/zstd_compress_literals.c @@ -92,16 +92,37 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* return flSize+1; } -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2, - unsigned suspectUncompressible, HUF_depth_mode depthMode) +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : 8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) { - size_t const minGain = ZSTD_minGain(srcSize, strategy); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); BYTE* const ostart = (BYTE*)dst; U32 singleStream = srcSize < 256; @@ -119,15 +140,14 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, if (disableLiteralCompression) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); { HUF_repeat repeat = prevHuf->repeatMode; int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0; + HUF_depth_mode const depthMode = (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD) ? HUF_depth_optimal : HUF_depth_fast; typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode); huf_compress_f huf_compress; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; @@ -146,10 +166,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, } } - if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } if (cLitSize==1) { ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h index e5da204c8..9eb74729d 100644 --- a/lib/compress/zstd_compress_literals.h +++ b/lib/compress/zstd_compress_literals.h @@ -18,14 +18,18 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize, void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2, - unsigned suspectUncompressible, HUF_depth_mode depthMode); + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); #endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 9b14093cb..68d24aa7b 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -16,7 +16,7 @@ #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ #define ZSTD_MAX_PRICE (1<<30) -#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ /*-************************************* @@ -96,14 +96,18 @@ static U32 sum_u32(const unsigned table[], size_t nbElts) return total; } -static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift) +typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e; + +static U32 +ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1) { U32 s, sum=0; DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift ); assert(shift < 30); for (s=0; s> shift); + unsigned const base = base1 ? 1 : (table[s]>0); + unsigned const newStat = base + (table[s] >> shift); sum += newStat; table[s] = newStat; } @@ -120,7 +124,7 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); assert(logTarget < 30); if (factor <= 1) return prevsum; - return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor)); + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); } /* ZSTD_rescaleFreqs() : @@ -202,14 +206,14 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, optPtr->offCodeSum += optPtr->offCodeFreq[of]; } } - } else { /* huf.repeatMode != HUF_repeat_valid => presumed not a dictionary */ + } else { /* first block, no dictionary */ assert(optPtr->litFreq != NULL); if (compressedLiterals) { /* base initial cost of literals on direct frequency within src */ unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8); + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); } { unsigned const baseLLfreqs[MaxLL+1] = { From ebba9ff4259d874030cce14c6a34e988e5679a50 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 3 Jan 2023 14:04:23 -0800 Subject: [PATCH 2/3] update regression results --- lib/compress/huf_compress.c | 17 ++-- tests/regression/results.csv | 190 +++++++++++++++++------------------ 2 files changed, 105 insertions(+), 102 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 007bb15e7..f7ca2d3bb 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -1262,20 +1262,23 @@ unsigned HUF_optimalTableLog( const unsigned* count, HUF_depth_mode depthMode) { - unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); - if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/ - BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + if (depthMode != HUF_depth_optimal) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); size_t maxBits, hSize, newSize; const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); const unsigned minTableLog = HUF_minTableLog(symbolCardinality); size_t optSize = ((size_t) ~0) - 1; - unsigned optLogGuess; + unsigned optLog = maxTableLog, optLogGuess; DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); - if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /* silently return if workspace is not large enough */ /* Search until size increases */ for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { @@ -1300,9 +1303,9 @@ unsigned HUF_optimalTableLog( optLog = optLogGuess; } } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; } - assert(optLog <= HUF_TABLELOG_MAX); - return optLog; } /* HUF_compress_internal() : diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 4a44dbb00..188eea900 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -11,10 +11,10 @@ silesia.tar, level 6, compress silesia.tar, level 7, compress simple, 4576661 silesia.tar, level 9, compress simple, 4552899 silesia.tar, level 13, compress simple, 4502956 -silesia.tar, level 16, compress simple, 4360527 -silesia.tar, level 19, compress simple, 4267021 +silesia.tar, level 16, compress simple, 4360546 +silesia.tar, level 19, compress simple, 4265911 silesia.tar, uncompressed literals, compress simple, 4854086 -silesia.tar, uncompressed literals optimal, compress simple, 4267021 +silesia.tar, uncompressed literals optimal, compress simple, 4265911 silesia.tar, huffman literals, compress simple, 6179047 github.tar, level -5, compress simple, 52115 github.tar, level -3, compress simple, 45678 @@ -28,10 +28,10 @@ github.tar, level 6, compress github.tar, level 7, compress simple, 38110 github.tar, level 9, compress simple, 36760 github.tar, level 13, compress simple, 35501 -github.tar, level 16, compress simple, 40471 -github.tar, level 19, compress simple, 32149 +github.tar, level 16, compress simple, 40466 +github.tar, level 19, compress simple, 32276 github.tar, uncompressed literals, compress simple, 38831 -github.tar, uncompressed literals optimal, compress simple, 32149 +github.tar, uncompressed literals optimal, compress simple, 32276 github.tar, huffman literals, compress simple, 42560 silesia, level -5, compress cctx, 6857372 silesia, level -3, compress cctx, 6503412 @@ -45,8 +45,8 @@ silesia, level 6, compress silesia, level 7, compress cctx, 4566984 silesia, level 9, compress cctx, 4543018 silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4359864 -silesia, level 19, compress cctx, 4296438 +silesia, level 16, compress cctx, 4360041 +silesia, level 19, compress cctx, 4296055 silesia, long distance mode, compress cctx, 4842075 silesia, multithreaded, compress cctx, 4842075 silesia, multithreaded long distance mode, compress cctx, 4842075 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912197 silesia, explicit params, compress cctx, 4794052 silesia, uncompressed literals, compress cctx, 4842075 -silesia, uncompressed literals optimal, compress cctx, 4296438 +silesia, uncompressed literals optimal, compress cctx, 4296055 silesia, huffman literals, compress cctx, 6172202 silesia, multithreaded with advanced params, compress cctx, 4842075 github, level -5, compress cctx, 204407 @@ -80,11 +80,11 @@ github, level 7, compress github, level 7 with dict, compress cctx, 38755 github, level 9, compress cctx, 135122 github, level 9 with dict, compress cctx, 39398 -github, level 13, compress cctx, 132729 +github, level 13, compress cctx, 132878 github, level 13 with dict, compress cctx, 39948 -github, level 16, compress cctx, 132729 +github, level 16, compress cctx, 133209 github, level 16 with dict, compress cctx, 37568 -github, level 19, compress cctx, 132729 +github, level 19, compress cctx, 132879 github, level 19 with dict, compress cctx, 37567 github, long distance mode, compress cctx, 141069 github, multithreaded, compress cctx, 141069 @@ -94,7 +94,7 @@ github, small hash log, compress github, small chain log, compress cctx, 139242 github, explicit params, compress cctx, 140932 github, uncompressed literals, compress cctx, 136332 -github, uncompressed literals optimal, compress cctx, 132729 +github, uncompressed literals optimal, compress cctx, 132879 github, huffman literals, compress cctx, 175468 github, multithreaded with advanced params, compress cctx, 141069 silesia, level -5, zstdcli, 6857420 @@ -109,8 +109,8 @@ silesia, level 6, zstdcli, silesia, level 7, zstdcli, 4567032 silesia, level 9, zstdcli, 4543066 silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4359912 -silesia, level 19, zstdcli, 4296486 +silesia, level 16, zstdcli, 4360089 +silesia, level 19, zstdcli, 4296103 silesia, long distance mode, zstdcli, 4833785 silesia, multithreaded, zstdcli, 4842123 silesia, multithreaded long distance mode, zstdcli, 4833785 @@ -134,14 +134,14 @@ silesia.tar, level 6, zstdcli, silesia.tar, level 7, zstdcli, 4578719 silesia.tar, level 9, zstdcli, 4552903 silesia.tar, level 13, zstdcli, 4502960 -silesia.tar, level 16, zstdcli, 4360531 -silesia.tar, level 19, zstdcli, 4267025 +silesia.tar, level 16, zstdcli, 4360550 +silesia.tar, level 19, zstdcli, 4265915 silesia.tar, no source size, zstdcli, 4854160 silesia.tar, long distance mode, zstdcli, 4845745 silesia.tar, multithreaded, zstdcli, 4854164 silesia.tar, multithreaded long distance mode, zstdcli, 4845745 silesia.tar, small window log, zstdcli, 7100701 -silesia.tar, small hash log, zstdcli, 6529289 +silesia.tar, small hash log, zstdcli, 6529264 silesia.tar, small chain log, zstdcli, 4917022 silesia.tar, explicit params, zstdcli, 4820713 silesia.tar, uncompressed literals, zstdcli, 5122571 @@ -170,11 +170,11 @@ github, level 7, zstdcli, github, level 7 with dict, zstdcli, 40745 github, level 9, zstdcli, 137122 github, level 9 with dict, zstdcli, 41393 -github, level 13, zstdcli, 134729 +github, level 13, zstdcli, 134878 github, level 13 with dict, zstdcli, 41900 -github, level 16, zstdcli, 134729 +github, level 16, zstdcli, 135209 github, level 16 with dict, zstdcli, 39577 -github, level 19, zstdcli, 134729 +github, level 19, zstdcli, 134879 github, level 19 with dict, zstdcli, 39576 github, long distance mode, zstdcli, 138332 github, multithreaded, zstdcli, 138332 @@ -184,7 +184,7 @@ github, small hash log, zstdcli, github, small chain log, zstdcli, 138341 github, explicit params, zstdcli, 136197 github, uncompressed literals, zstdcli, 167911 -github, uncompressed literals optimal, zstdcli, 159227 +github, uncompressed literals optimal, zstdcli, 154667 github, huffman literals, zstdcli, 144365 github, multithreaded with advanced params, zstdcli, 167911 github.tar, level -5, zstdcli, 52119 @@ -211,9 +211,9 @@ github.tar, level 9, zstdcli, github.tar, level 9 with dict, zstdcli, 36632 github.tar, level 13, zstdcli, 35505 github.tar, level 13 with dict, zstdcli, 37134 -github.tar, level 16, zstdcli, 40475 +github.tar, level 16, zstdcli, 40470 github.tar, level 16 with dict, zstdcli, 33378 -github.tar, level 19, zstdcli, 32153 +github.tar, level 19, zstdcli, 32280 github.tar, level 19 with dict, zstdcli, 32716 github.tar, no source size, zstdcli, 38832 github.tar, no source size with dict, zstdcli, 38004 @@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass, 4505046 silesia, level 12 row 2, advanced one pass, 4503116 silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4359864 -silesia, level 19, advanced one pass, 4296438 +silesia, level 16, advanced one pass, 4360041 +silesia, level 19, advanced one pass, 4296055 silesia, no source size, advanced one pass, 4842075 silesia, long distance mode, advanced one pass, 4833710 silesia, multithreaded, advanced one pass, 4842075 @@ -282,14 +282,14 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass, 4514049 silesia.tar, level 12 row 2, advanced one pass, 4513797 silesia.tar, level 13, advanced one pass, 4502956 -silesia.tar, level 16, advanced one pass, 4360527 -silesia.tar, level 19, advanced one pass, 4267021 +silesia.tar, level 16, advanced one pass, 4360546 +silesia.tar, level 19, advanced one pass, 4265911 silesia.tar, no source size, advanced one pass, 4854086 silesia.tar, long distance mode, advanced one pass, 4840452 silesia.tar, multithreaded, advanced one pass, 4854160 silesia.tar, multithreaded long distance mode, advanced one pass, 4845741 silesia.tar, small window log, advanced one pass, 7100655 -silesia.tar, small hash log, advanced one pass, 6529231 +silesia.tar, small hash log, advanced one pass, 6529206 silesia.tar, small chain log, advanced one pass, 4917041 silesia.tar, explicit params, advanced one pass, 4806855 silesia.tar, uncompressed literals, advanced one pass, 5122473 @@ -370,39 +370,39 @@ github, level 9 with dict dms, advanced github, level 9 with dict dds, advanced one pass, 39393 github, level 9 with dict copy, advanced one pass, 39398 github, level 9 with dict load, advanced one pass, 41710 -github, level 11 row 1, advanced one pass, 135119 +github, level 11 row 1, advanced one pass, 135367 github, level 11 row 1 with dict dms, advanced one pass, 39671 github, level 11 row 1 with dict dds, advanced one pass, 39671 github, level 11 row 1 with dict copy, advanced one pass, 39651 github, level 11 row 1 with dict load, advanced one pass, 41360 -github, level 11 row 2, advanced one pass, 135119 +github, level 11 row 2, advanced one pass, 135367 github, level 11 row 2 with dict dms, advanced one pass, 39671 github, level 11 row 2 with dict dds, advanced one pass, 39671 github, level 11 row 2 with dict copy, advanced one pass, 39651 github, level 11 row 2 with dict load, advanced one pass, 41360 -github, level 12 row 1, advanced one pass, 134180 +github, level 12 row 1, advanced one pass, 134402 github, level 12 row 1 with dict dms, advanced one pass, 39677 github, level 12 row 1 with dict dds, advanced one pass, 39677 github, level 12 row 1 with dict copy, advanced one pass, 39677 github, level 12 row 1 with dict load, advanced one pass, 41166 -github, level 12 row 2, advanced one pass, 134180 +github, level 12 row 2, advanced one pass, 134402 github, level 12 row 2 with dict dms, advanced one pass, 39677 github, level 12 row 2 with dict dds, advanced one pass, 39677 github, level 12 row 2 with dict copy, advanced one pass, 39677 github, level 12 row 2 with dict load, advanced one pass, 41166 -github, level 13, advanced one pass, 132729 +github, level 13, advanced one pass, 132878 github, level 13 with dict, advanced one pass, 39900 github, level 13 with dict dms, advanced one pass, 39900 github, level 13 with dict dds, advanced one pass, 39900 github, level 13 with dict copy, advanced one pass, 39948 github, level 13 with dict load, advanced one pass, 42624 -github, level 16, advanced one pass, 132729 +github, level 16, advanced one pass, 133209 github, level 16 with dict, advanced one pass, 37577 github, level 16 with dict dms, advanced one pass, 37577 github, level 16 with dict dds, advanced one pass, 37577 github, level 16 with dict copy, advanced one pass, 37568 github, level 16 with dict load, advanced one pass, 42338 -github, level 19, advanced one pass, 132729 +github, level 19, advanced one pass, 132879 github, level 19 with dict, advanced one pass, 37576 github, level 19 with dict dms, advanced one pass, 37576 github, level 19 with dict dds, advanced one pass, 37576 @@ -418,7 +418,7 @@ github, small hash log, advanced github, small chain log, advanced one pass, 136341 github, explicit params, advanced one pass, 137727 github, uncompressed literals, advanced one pass, 165911 -github, uncompressed literals optimal, advanced one pass, 157227 +github, uncompressed literals optimal, advanced one pass, 152667 github, huffman literals, advanced one pass, 142365 github, multithreaded with advanced params, advanced one pass, 165911 github.tar, level -5, advanced one pass, 52115 @@ -521,13 +521,13 @@ github.tar, level 13 with dict dms, advanced github.tar, level 13 with dict dds, advanced one pass, 37220 github.tar, level 13 with dict copy, advanced one pass, 37130 github.tar, level 13 with dict load, advanced one pass, 36010 -github.tar, level 16, advanced one pass, 40471 +github.tar, level 16, advanced one pass, 40466 github.tar, level 16 with dict, advanced one pass, 33374 github.tar, level 16 with dict dms, advanced one pass, 33206 github.tar, level 16 with dict dds, advanced one pass, 33206 github.tar, level 16 with dict copy, advanced one pass, 33374 github.tar, level 16 with dict load, advanced one pass, 39081 -github.tar, level 19, advanced one pass, 32149 +github.tar, level 19, advanced one pass, 32276 github.tar, level 19 with dict, advanced one pass, 32712 github.tar, level 19 with dict dms, advanced one pass, 32555 github.tar, level 19 with dict dds, advanced one pass, 32555 @@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass small out, 4505046 silesia, level 12 row 2, advanced one pass small out, 4503116 silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4359864 -silesia, level 19, advanced one pass small out, 4296438 +silesia, level 16, advanced one pass small out, 4360041 +silesia, level 19, advanced one pass small out, 4296055 silesia, no source size, advanced one pass small out, 4842075 silesia, long distance mode, advanced one pass small out, 4833710 silesia, multithreaded, advanced one pass small out, 4842075 @@ -600,14 +600,14 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass small out, 4514049 silesia.tar, level 12 row 2, advanced one pass small out, 4513797 silesia.tar, level 13, advanced one pass small out, 4502956 -silesia.tar, level 16, advanced one pass small out, 4360527 -silesia.tar, level 19, advanced one pass small out, 4267021 +silesia.tar, level 16, advanced one pass small out, 4360546 +silesia.tar, level 19, advanced one pass small out, 4265911 silesia.tar, no source size, advanced one pass small out, 4854086 silesia.tar, long distance mode, advanced one pass small out, 4840452 silesia.tar, multithreaded, advanced one pass small out, 4854160 silesia.tar, multithreaded long distance mode, advanced one pass small out, 4845741 silesia.tar, small window log, advanced one pass small out, 7100655 -silesia.tar, small hash log, advanced one pass small out, 6529231 +silesia.tar, small hash log, advanced one pass small out, 6529206 silesia.tar, small chain log, advanced one pass small out, 4917041 silesia.tar, explicit params, advanced one pass small out, 4806855 silesia.tar, uncompressed literals, advanced one pass small out, 5122473 @@ -688,39 +688,39 @@ github, level 9 with dict dms, advanced github, level 9 with dict dds, advanced one pass small out, 39393 github, level 9 with dict copy, advanced one pass small out, 39398 github, level 9 with dict load, advanced one pass small out, 41710 -github, level 11 row 1, advanced one pass small out, 135119 +github, level 11 row 1, advanced one pass small out, 135367 github, level 11 row 1 with dict dms, advanced one pass small out, 39671 github, level 11 row 1 with dict dds, advanced one pass small out, 39671 github, level 11 row 1 with dict copy, advanced one pass small out, 39651 github, level 11 row 1 with dict load, advanced one pass small out, 41360 -github, level 11 row 2, advanced one pass small out, 135119 +github, level 11 row 2, advanced one pass small out, 135367 github, level 11 row 2 with dict dms, advanced one pass small out, 39671 github, level 11 row 2 with dict dds, advanced one pass small out, 39671 github, level 11 row 2 with dict copy, advanced one pass small out, 39651 github, level 11 row 2 with dict load, advanced one pass small out, 41360 -github, level 12 row 1, advanced one pass small out, 134180 +github, level 12 row 1, advanced one pass small out, 134402 github, level 12 row 1 with dict dms, advanced one pass small out, 39677 github, level 12 row 1 with dict dds, advanced one pass small out, 39677 github, level 12 row 1 with dict copy, advanced one pass small out, 39677 github, level 12 row 1 with dict load, advanced one pass small out, 41166 -github, level 12 row 2, advanced one pass small out, 134180 +github, level 12 row 2, advanced one pass small out, 134402 github, level 12 row 2 with dict dms, advanced one pass small out, 39677 github, level 12 row 2 with dict dds, advanced one pass small out, 39677 github, level 12 row 2 with dict copy, advanced one pass small out, 39677 github, level 12 row 2 with dict load, advanced one pass small out, 41166 -github, level 13, advanced one pass small out, 132729 +github, level 13, advanced one pass small out, 132878 github, level 13 with dict, advanced one pass small out, 39900 github, level 13 with dict dms, advanced one pass small out, 39900 github, level 13 with dict dds, advanced one pass small out, 39900 github, level 13 with dict copy, advanced one pass small out, 39948 github, level 13 with dict load, advanced one pass small out, 42624 -github, level 16, advanced one pass small out, 132729 +github, level 16, advanced one pass small out, 133209 github, level 16 with dict, advanced one pass small out, 37577 github, level 16 with dict dms, advanced one pass small out, 37577 github, level 16 with dict dds, advanced one pass small out, 37577 github, level 16 with dict copy, advanced one pass small out, 37568 github, level 16 with dict load, advanced one pass small out, 42338 -github, level 19, advanced one pass small out, 132729 +github, level 19, advanced one pass small out, 132879 github, level 19 with dict, advanced one pass small out, 37576 github, level 19 with dict dms, advanced one pass small out, 37576 github, level 19 with dict dds, advanced one pass small out, 37576 @@ -736,7 +736,7 @@ github, small hash log, advanced github, small chain log, advanced one pass small out, 136341 github, explicit params, advanced one pass small out, 137727 github, uncompressed literals, advanced one pass small out, 165911 -github, uncompressed literals optimal, advanced one pass small out, 157227 +github, uncompressed literals optimal, advanced one pass small out, 152667 github, huffman literals, advanced one pass small out, 142365 github, multithreaded with advanced params, advanced one pass small out, 165911 github.tar, level -5, advanced one pass small out, 52115 @@ -839,13 +839,13 @@ github.tar, level 13 with dict dms, advanced github.tar, level 13 with dict dds, advanced one pass small out, 37220 github.tar, level 13 with dict copy, advanced one pass small out, 37130 github.tar, level 13 with dict load, advanced one pass small out, 36010 -github.tar, level 16, advanced one pass small out, 40471 +github.tar, level 16, advanced one pass small out, 40466 github.tar, level 16 with dict, advanced one pass small out, 33374 github.tar, level 16 with dict dms, advanced one pass small out, 33206 github.tar, level 16 with dict dds, advanced one pass small out, 33206 github.tar, level 16 with dict copy, advanced one pass small out, 33374 github.tar, level 16 with dict load, advanced one pass small out, 39081 -github.tar, level 19, advanced one pass small out, 32149 +github.tar, level 19, advanced one pass small out, 32276 github.tar, level 19 with dict, advanced one pass small out, 32712 github.tar, level 19 with dict dms, advanced one pass small out, 32555 github.tar, level 19 with dict dds, advanced one pass small out, 32555 @@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced streaming, 4505046 silesia, level 12 row 2, advanced streaming, 4503116 silesia, level 13, advanced streaming, 4493990 -silesia, level 16, advanced streaming, 4359864 -silesia, level 19, advanced streaming, 4296438 +silesia, level 16, advanced streaming, 4360041 +silesia, level 19, advanced streaming, 4296055 silesia, no source size, advanced streaming, 4842039 silesia, long distance mode, advanced streaming, 4833710 silesia, multithreaded, advanced streaming, 4842075 @@ -918,14 +918,14 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced streaming, 4514049 silesia.tar, level 12 row 2, advanced streaming, 4513797 silesia.tar, level 13, advanced streaming, 4502956 -silesia.tar, level 16, advanced streaming, 4360527 -silesia.tar, level 19, advanced streaming, 4267021 +silesia.tar, level 16, advanced streaming, 4360546 +silesia.tar, level 19, advanced streaming, 4265911 silesia.tar, no source size, advanced streaming, 4859267 silesia.tar, long distance mode, advanced streaming, 4840452 silesia.tar, multithreaded, advanced streaming, 4854160 silesia.tar, multithreaded long distance mode, advanced streaming, 4845741 silesia.tar, small window log, advanced streaming, 7117559 -silesia.tar, small hash log, advanced streaming, 6529234 +silesia.tar, small hash log, advanced streaming, 6529209 silesia.tar, small chain log, advanced streaming, 4917021 silesia.tar, explicit params, advanced streaming, 4806873 silesia.tar, uncompressed literals, advanced streaming, 5127423 @@ -1006,39 +1006,39 @@ github, level 9 with dict dms, advanced github, level 9 with dict dds, advanced streaming, 39393 github, level 9 with dict copy, advanced streaming, 39398 github, level 9 with dict load, advanced streaming, 41710 -github, level 11 row 1, advanced streaming, 135119 +github, level 11 row 1, advanced streaming, 135367 github, level 11 row 1 with dict dms, advanced streaming, 39671 github, level 11 row 1 with dict dds, advanced streaming, 39671 github, level 11 row 1 with dict copy, advanced streaming, 39651 github, level 11 row 1 with dict load, advanced streaming, 41360 -github, level 11 row 2, advanced streaming, 135119 +github, level 11 row 2, advanced streaming, 135367 github, level 11 row 2 with dict dms, advanced streaming, 39671 github, level 11 row 2 with dict dds, advanced streaming, 39671 github, level 11 row 2 with dict copy, advanced streaming, 39651 github, level 11 row 2 with dict load, advanced streaming, 41360 -github, level 12 row 1, advanced streaming, 134180 +github, level 12 row 1, advanced streaming, 134402 github, level 12 row 1 with dict dms, advanced streaming, 39677 github, level 12 row 1 with dict dds, advanced streaming, 39677 github, level 12 row 1 with dict copy, advanced streaming, 39677 github, level 12 row 1 with dict load, advanced streaming, 41166 -github, level 12 row 2, advanced streaming, 134180 +github, level 12 row 2, advanced streaming, 134402 github, level 12 row 2 with dict dms, advanced streaming, 39677 github, level 12 row 2 with dict dds, advanced streaming, 39677 github, level 12 row 2 with dict copy, advanced streaming, 39677 github, level 12 row 2 with dict load, advanced streaming, 41166 -github, level 13, advanced streaming, 132729 +github, level 13, advanced streaming, 132878 github, level 13 with dict, advanced streaming, 39900 github, level 13 with dict dms, advanced streaming, 39900 github, level 13 with dict dds, advanced streaming, 39900 github, level 13 with dict copy, advanced streaming, 39948 github, level 13 with dict load, advanced streaming, 42624 -github, level 16, advanced streaming, 132729 +github, level 16, advanced streaming, 133209 github, level 16 with dict, advanced streaming, 37577 github, level 16 with dict dms, advanced streaming, 37577 github, level 16 with dict dds, advanced streaming, 37577 github, level 16 with dict copy, advanced streaming, 37568 github, level 16 with dict load, advanced streaming, 42338 -github, level 19, advanced streaming, 132729 +github, level 19, advanced streaming, 132879 github, level 19 with dict, advanced streaming, 37576 github, level 19 with dict dms, advanced streaming, 37576 github, level 19 with dict dds, advanced streaming, 37576 @@ -1054,7 +1054,7 @@ github, small hash log, advanced github, small chain log, advanced streaming, 136341 github, explicit params, advanced streaming, 137727 github, uncompressed literals, advanced streaming, 165911 -github, uncompressed literals optimal, advanced streaming, 157227 +github, uncompressed literals optimal, advanced streaming, 152667 github, huffman literals, advanced streaming, 142365 github, multithreaded with advanced params, advanced streaming, 165911 github.tar, level -5, advanced streaming, 52152 @@ -1157,13 +1157,13 @@ github.tar, level 13 with dict dms, advanced github.tar, level 13 with dict dds, advanced streaming, 37220 github.tar, level 13 with dict copy, advanced streaming, 37130 github.tar, level 13 with dict load, advanced streaming, 36010 -github.tar, level 16, advanced streaming, 40471 +github.tar, level 16, advanced streaming, 40466 github.tar, level 16 with dict, advanced streaming, 33374 github.tar, level 16 with dict dms, advanced streaming, 33206 github.tar, level 16 with dict dds, advanced streaming, 33206 github.tar, level 16 with dict copy, advanced streaming, 33374 github.tar, level 16 with dict load, advanced streaming, 39081 -github.tar, level 19, advanced streaming, 32149 +github.tar, level 19, advanced streaming, 32276 github.tar, level 19 with dict, advanced streaming, 32712 github.tar, level 19 with dict dms, advanced streaming, 32555 github.tar, level 19 with dict dds, advanced streaming, 32555 @@ -1194,11 +1194,11 @@ silesia, level 6, old stre silesia, level 7, old streaming, 4566984 silesia, level 9, old streaming, 4543018 silesia, level 13, old streaming, 4493990 -silesia, level 16, old streaming, 4359864 -silesia, level 19, old streaming, 4296438 +silesia, level 16, old streaming, 4360041 +silesia, level 19, old streaming, 4296055 silesia, no source size, old streaming, 4842039 silesia, uncompressed literals, old streaming, 4842075 -silesia, uncompressed literals optimal, old streaming, 4296438 +silesia, uncompressed literals optimal, old streaming, 4296055 silesia, huffman literals, old streaming, 6172207 silesia.tar, level -5, old streaming, 6856523 silesia.tar, level -3, old streaming, 6505954 @@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming, 4576664 silesia.tar, level 9, old streaming, 4552900 silesia.tar, level 13, old streaming, 4502956 -silesia.tar, level 16, old streaming, 4360527 -silesia.tar, level 19, old streaming, 4267021 +silesia.tar, level 16, old streaming, 4360546 +silesia.tar, level 19, old streaming, 4265911 silesia.tar, no source size, old streaming, 4859267 silesia.tar, uncompressed literals, old streaming, 4859271 -silesia.tar, uncompressed literals optimal, old streaming, 4267021 +silesia.tar, uncompressed literals optimal, old streaming, 4265911 silesia.tar, huffman literals, old streaming, 6179056 github, level -5, old streaming, 204407 github, level -5 with dict, old streaming, 46718 @@ -1240,16 +1240,16 @@ github, level 7, old stre github, level 7 with dict, old streaming, 38758 github, level 9, old streaming, 135122 github, level 9 with dict, old streaming, 39437 -github, level 13, old streaming, 132729 +github, level 13, old streaming, 132878 github, level 13 with dict, old streaming, 39900 -github, level 16, old streaming, 132729 +github, level 16, old streaming, 133209 github, level 16 with dict, old streaming, 37577 -github, level 19, old streaming, 132729 +github, level 19, old streaming, 132879 github, level 19 with dict, old streaming, 37576 github, no source size, old streaming, 140599 github, no source size with dict, old streaming, 40654 github, uncompressed literals, old streaming, 136332 -github, uncompressed literals optimal, old streaming, 132729 +github, uncompressed literals optimal, old streaming, 132879 github, huffman literals, old streaming, 175468 github.tar, level -5, old streaming, 52152 github.tar, level -5 with dict, old streaming, 51045 @@ -1275,14 +1275,14 @@ github.tar, level 9, old stre github.tar, level 9 with dict, old streaming, 36484 github.tar, level 13, old streaming, 35501 github.tar, level 13 with dict, old streaming, 37130 -github.tar, level 16, old streaming, 40471 +github.tar, level 16, old streaming, 40466 github.tar, level 16 with dict, old streaming, 33374 -github.tar, level 19, old streaming, 32149 +github.tar, level 19, old streaming, 32276 github.tar, level 19 with dict, old streaming, 32712 github.tar, no source size, old streaming, 38828 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38831 -github.tar, uncompressed literals optimal, old streaming, 32149 +github.tar, uncompressed literals optimal, old streaming, 32276 github.tar, huffman literals, old streaming, 42560 silesia, level -5, old streaming advanced, 6854744 silesia, level -3, old streaming advanced, 6503319 @@ -1296,8 +1296,8 @@ silesia, level 6, old stre silesia, level 7, old streaming advanced, 4566984 silesia, level 9, old streaming advanced, 4543018 silesia, level 13, old streaming advanced, 4493990 -silesia, level 16, old streaming advanced, 4359864 -silesia, level 19, old streaming advanced, 4296438 +silesia, level 16, old streaming advanced, 4360041 +silesia, level 19, old streaming advanced, 4296055 silesia, no source size, old streaming advanced, 4842039 silesia, long distance mode, old streaming advanced, 4842075 silesia, multithreaded, old streaming advanced, 4842075 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912197 silesia, explicit params, old streaming advanced, 4795452 silesia, uncompressed literals, old streaming advanced, 4842075 -silesia, uncompressed literals optimal, old streaming advanced, 4296438 +silesia, uncompressed literals optimal, old streaming advanced, 4296055 silesia, huffman literals, old streaming advanced, 6172207 silesia, multithreaded with advanced params, old streaming advanced, 4842075 silesia.tar, level -5, old streaming advanced, 6856523 @@ -1322,18 +1322,18 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming advanced, 4576664 silesia.tar, level 9, old streaming advanced, 4552900 silesia.tar, level 13, old streaming advanced, 4502956 -silesia.tar, level 16, old streaming advanced, 4360527 -silesia.tar, level 19, old streaming advanced, 4267021 +silesia.tar, level 16, old streaming advanced, 4360546 +silesia.tar, level 19, old streaming advanced, 4265911 silesia.tar, no source size, old streaming advanced, 4859267 silesia.tar, long distance mode, old streaming advanced, 4859271 silesia.tar, multithreaded, old streaming advanced, 4859271 silesia.tar, multithreaded long distance mode, old streaming advanced, 4859271 silesia.tar, small window log, old streaming advanced, 7117562 -silesia.tar, small hash log, old streaming advanced, 6529234 +silesia.tar, small hash log, old streaming advanced, 6529209 silesia.tar, small chain log, old streaming advanced, 4917021 silesia.tar, explicit params, old streaming advanced, 4806873 silesia.tar, uncompressed literals, old streaming advanced, 4859271 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4267021 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4265911 silesia.tar, huffman literals, old streaming advanced, 6179056 silesia.tar, multithreaded with advanced params, old streaming advanced, 4859271 github, level -5, old streaming advanced, 213265 @@ -1360,9 +1360,9 @@ github, level 9, old stre github, level 9 with dict, old streaming advanced, 38981 github, level 13, old streaming advanced, 138676 github, level 13 with dict, old streaming advanced, 39725 -github, level 16, old streaming advanced, 138676 +github, level 16, old streaming advanced, 138575 github, level 16 with dict, old streaming advanced, 40789 -github, level 19, old streaming advanced, 132729 +github, level 19, old streaming advanced, 132879 github, level 19 with dict, old streaming advanced, 37576 github, no source size, old streaming advanced, 140599 github, no source size with dict, old streaming advanced, 40608 @@ -1374,7 +1374,7 @@ github, small hash log, old stre github, small chain log, old streaming advanced, 139275 github, explicit params, old streaming advanced, 140937 github, uncompressed literals, old streaming advanced, 141104 -github, uncompressed literals optimal, old streaming advanced, 132729 +github, uncompressed literals optimal, old streaming advanced, 132879 github, huffman literals, old streaming advanced, 181107 github, multithreaded with advanced params, old streaming advanced, 141104 github.tar, level -5, old streaming advanced, 52152 @@ -1401,9 +1401,9 @@ github.tar, level 9, old stre github.tar, level 9 with dict, old streaming advanced, 36312 github.tar, level 13, old streaming advanced, 35501 github.tar, level 13 with dict, old streaming advanced, 35807 -github.tar, level 16, old streaming advanced, 40471 +github.tar, level 16, old streaming advanced, 40466 github.tar, level 16 with dict, old streaming advanced, 38578 -github.tar, level 19, old streaming advanced, 32149 +github.tar, level 19, old streaming advanced, 32276 github.tar, level 19 with dict, old streaming advanced, 32704 github.tar, no source size, old streaming advanced, 38828 github.tar, no source size with dict, old streaming advanced, 38015 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41385 github.tar, uncompressed literals, old streaming advanced, 38831 -github.tar, uncompressed literals optimal, old streaming advanced, 32149 +github.tar, uncompressed literals optimal, old streaming advanced, 32276 github.tar, huffman literals, old streaming advanced, 42560 github.tar, multithreaded with advanced params, old streaming advanced, 38831 github, level -5 with dict, old streaming cdict, 46718 From c79fb4d78d986f5e85cd79370ecf1cccec473261 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2022 16:21:29 -0800 Subject: [PATCH 3/3] update levels.sh test comparing level 19 to level 22 and expecting a stricter better result from level 22 is not that guaranteed, because level 19 and 22 are very close to each other, especially for small files, so any noise in the final compression result result in failing this test. Level 22 could be compared to something much lower, like level 15, But level 19 is required anyway, because there is a clamping test which depends on it. Removed level 22, kept level 19 --- tests/cli-tests/compression/levels.sh | 4 +--- tests/cli-tests/compression/levels.sh.stderr.exact | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh index 4837790ce..47ac2029f 100755 --- a/tests/cli-tests/compression/levels.sh +++ b/tests/cli-tests/compression/levels.sh @@ -10,11 +10,9 @@ zstd --fast=10 file -o file-f10.zst zstd --fast=1 file -o file-f1.zst zstd -1 file -o file-1.zst zstd -19 file -o file-19.zst -zstd -22 --ultra file -o file-22.zst -zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst +zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst -cmp_size -ne file-19.zst file-22.zst cmp_size -lt file-19.zst file-1.zst cmp_size -lt file-1.zst file-f1.zst cmp_size -lt file-f1.zst file-f10.zst diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact index cb00433e6..5664f854b 100644 --- a/tests/cli-tests/compression/levels.sh.stderr.exact +++ b/tests/cli-tests/compression/levels.sh.stderr.exact @@ -6,11 +6,9 @@ zstd --fast=10 file -o file-f10.zst zstd --fast=1 file -o file-f1.zst zstd -1 file -o file-1.zst zstd -19 file -o file-19.zst -zstd -22 --ultra file -o file-22.zst -zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst +zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst -cmp_size -ne file-19.zst file-22.zst cmp_size -lt file-19.zst file-1.zst cmp_size -lt file-1.zst file-f1.zst cmp_size -lt file-f1.zst file-f10.zst