ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2771 from facebook/opt_investigation

Browse Source 
Improve optimal parser performance on small data
This commit is contained in:
Yann Collet
2021-09-14 10:36:34 -07:00
committed by GitHub
parent d22bbed5db fd94b9d1c9
commit 2e6f5bc0d8
10 changed files with 257 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_opt.c
+64 -57
View File
@@ -14,7 +14,6 @@
#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */
#define ZSTD_MAX_PRICE (1<<30) #define ZSTD_MAX_PRICE (1<<30)
#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ #define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
@@ -24,11 +23,11 @@
* Price functions for optimal parser * Price functions for optimal parser
***************************************/ ***************************************/
#if 0 /* approximation at bit level */ #if 0 /* approximation at bit level (for tests) */
# define BITCOST_ACCURACY 0 # define BITCOST_ACCURACY 0
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) # define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat))
#elif 0 /* fractional bit accuracy */ #elif 0 /* fractional bit accuracy (for tests) */
# define BITCOST_ACCURACY 8 # define BITCOST_ACCURACY 8
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) # define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat))
@@ -79,25 +78,46 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
} }
/* ZSTD_downscaleStat() : static U32 sum_u32(const unsigned table[], size_t nbElts)
* reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) {
* return the resulting sum of elements */ size_t n;
static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) U32 total = 0;
for (n=0; n<nbElts; n++) {
total += table[n];
}
return total;
}
static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift)
{ {
U32 s, sum=0; U32 s, sum=0;
DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1); DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift);
assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31); assert(shift < 30);
for (s=0; s<lastEltIndex+1; s++) { for (s=0; s<lastEltIndex+1; s++) {
table[s] = 1 + (table[s] >> (ZSTD_FREQ_DIV+malus)); table[s] = 1 + (table[s] >> shift);
sum += table[s]; sum += table[s];
} }
return sum; return sum;
} }
/* ZSTD_scaleStats() :
* reduce all elements in table is sum too large
* return the resulting sum of elements */
static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
{
U32 const prevsum = sum_u32(table, lastEltIndex+1);
U32 const factor = prevsum >> logTarget;
DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
assert(logTarget < 30);
if (factor <= 1) return prevsum;
return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor));
}
/* ZSTD_rescaleFreqs() : /* ZSTD_rescaleFreqs() :
* if first block (detected by optPtr->litLengthSum == 0) : init statistics * if first block (detected by optPtr->litLengthSum == 0) : init statistics
* take hints from dictionary if there is one * take hints from dictionary if there is one
* or init from zero, using src for literals stats, or flat 1 for match symbols * and init from zero if there is none,
* using src for literals stats, and baseline stats for sequence symbols
* otherwise downscale existing stats, to be used as seed for next block. * otherwise downscale existing stats, to be used as seed for next block.
*/ */
static void static void
@@ -174,14 +194,18 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
if (compressedLiterals) { if (compressedLiterals) {
unsigned lit = MaxLit; unsigned lit = MaxLit;
HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
} }
{ unsigned ll; { unsigned const baseLLfreqs[MaxLL+1] = {
for (ll=0; ll<=MaxLL; ll++) 4, 2, 1, 1, 1, 1, 1, 1,
optPtr->litLengthFreq[ll] = 1; 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1
};
ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
} }
optPtr->litLengthSum = MaxLL+1;
{ unsigned ml; { unsigned ml;
for (ml=0; ml<=MaxML; ml++) for (ml=0; ml<=MaxML; ml++)
@@ -189,21 +213,25 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
} }
optPtr->matchLengthSum = MaxML+1; optPtr->matchLengthSum = MaxML+1;
{ unsigned of; { unsigned const baseOFCfreqs[MaxOff+1] = {
for (of=0; of<=MaxOff; of++) 6, 2, 1, 1, 2, 3, 4, 4,
optPtr->offCodeFreq[of] = 1; 4, 3, 2, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1
};
ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
} }
optPtr->offCodeSum = MaxOff+1;
} }
} else { /* new block : re-use previous statistics, scaled down */ } else { /* new block : re-use previous statistics, scaled down */
if (compressedLiterals) if (compressedLiterals)
optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11);
optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11);
optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11);
} }
ZSTD_setBasePrices(optPtr, optLevel); ZSTD_setBasePrices(optPtr, optLevel);
@@ -901,11 +929,11 @@ static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
} }
/*-******************************* /*-*******************************
* Optimal parser * Optimal parser
*********************************/ *********************************/
static U32 ZSTD_totalLen(ZSTD_optimal_t sol) static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
{ {
return sol.litlen + sol.mlen; return sol.litlen + sol.mlen;
@@ -987,7 +1015,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
* in every price. We include the literal length to avoid negative * in every price. We include the literal length to avoid negative
* prices when we subtract the previous literal length. * prices when we subtract the previous literal length.
*/ */
opt[0].price = ZSTD_litLengthPrice(litlen, optStatePtr, optLevel); opt[0].price = (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel);
/* large match -> immediate encoding */ /* large match -> immediate encoding */
{ U32 const maxML = matches[nbMatches-1].len; { U32 const maxML = matches[nbMatches-1].len;
@@ -1007,7 +1035,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
} } } }
/* set prices for first matches starting position == 0 */ /* set prices for first matches starting position == 0 */
{ U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); assert(opt[0].price >= 0);
{ U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
U32 pos; U32 pos;
U32 matchNb; U32 matchNb;
for (pos = 1; pos < minMatch; pos++) { for (pos = 1; pos < minMatch; pos++) {
@@ -1024,7 +1053,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
opt[pos].mlen = pos; opt[pos].mlen = pos;
opt[pos].off = offset; opt[pos].off = offset;
opt[pos].litlen = litlen; opt[pos].litlen = litlen;
opt[pos].price = sequencePrice; opt[pos].price = (int)sequencePrice;
} } } }
last_pos = pos-1; last_pos = pos-1;
} }
@@ -1039,9 +1068,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
/* Fix current position with one literal if cheaper */ /* Fix current position with one literal if cheaper */
{ U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1;
int const price = opt[cur-1].price int const price = opt[cur-1].price
+ ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel)
+ ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel)
- ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel);
assert(price < 1000000000); /* overflow check */ assert(price < 1000000000); /* overflow check */
if (price <= opt[cur].price) { if (price <= opt[cur].price) {
DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
@@ -1084,9 +1113,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
} }
assert(opt[cur].price >= 0);
{ U32 const ll0 = (opt[cur].mlen != 0); { U32 const ll0 = (opt[cur].mlen != 0);
U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
U32 const previousPrice = opt[cur].price; U32 const previousPrice = (U32)opt[cur].price;
U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
U32 matchNb; U32 matchNb;
@@ -1126,7 +1156,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */
U32 const pos = cur + mlen; U32 const pos = cur + mlen;
int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
if ((pos > last_pos) || (price < opt[pos].price)) { if ((pos > last_pos) || (price < opt[pos].price)) {
DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)",
@@ -1222,28 +1252,7 @@ size_t ZSTD_compressBlock_btopt(
} }
/* used in 2-pass strategy */
static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
{
U32 s, sum=0;
assert(ZSTD_FREQ_DIV+bonus >= 0);
for (s=0; s<lastEltIndex+1; s++) {
table[s] <<= ZSTD_FREQ_DIV+bonus;
table[s]--;
sum += table[s];
}
return sum;
}
/* used in 2-pass strategy */
MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
{
if (ZSTD_compressedLiterals(optPtr))
optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
}
/* ZSTD_initStats_ultra(): /* ZSTD_initStats_ultra():
* make a first compression pass, just to seed stats with more accurate starting values. * make a first compression pass, just to seed stats with more accurate starting values.
@@ -1274,8 +1283,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
ms->window.lowLimit = ms->window.dictLimit; ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
} }
size_t ZSTD_compressBlock_btultra( size_t ZSTD_compressBlock_btultra(
lib/dictBuilder/cover.c
+2 -2
View File
@@ -47,7 +47,7 @@
* Console display * Console display
***************************************/ ***************************************/
#ifndef LOCALDISPLAYLEVEL #ifndef LOCALDISPLAYLEVEL
static int g_displayLevel = 2; static int g_displayLevel = 0;
#endif #endif
#undef DISPLAY #undef DISPLAY
#define DISPLAY(...) \ #define DISPLAY(...) \
@@ -735,7 +735,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
COVER_map_t activeDmers; COVER_map_t activeDmers;
parameters.splitPoint = 1.0; parameters.splitPoint = 1.0;
/* Initialize global data */ /* Initialize global data */
g_displayLevel = parameters.zParams.notificationLevel; g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Checks */ /* Checks */
if (!COVER_checkParameters(parameters, dictBufferCapacity)) { if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n"); DISPLAYLEVEL(1, "Cover parameters incorrect\n");
lib/dictBuilder/fastcover.c
+4 -4
View File
@@ -44,7 +44,7 @@
* Console display * Console display
***************************************/ ***************************************/
#ifndef LOCALDISPLAYLEVEL #ifndef LOCALDISPLAYLEVEL
static int g_displayLevel = 2; static int g_displayLevel = 0;
#endif #endif
#undef DISPLAY #undef DISPLAY
#define DISPLAY(...) \ #define DISPLAY(...) \
@@ -549,7 +549,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
ZDICT_cover_params_t coverParams; ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams; FASTCOVER_accel_t accelParams;
/* Initialize global data */ /* Initialize global data */
g_displayLevel = parameters.zParams.notificationLevel; g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Assign splitPoint and f if not provided */ /* Assign splitPoint and f if not provided */
parameters.splitPoint = 1.0; parameters.splitPoint = 1.0;
parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
@@ -632,7 +632,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel;
const unsigned shrinkDict = 0; const unsigned shrinkDict = 0;
/* Local variables */ /* Local variables */
const int displayLevel = parameters->zParams.notificationLevel; const int displayLevel = (int)parameters->zParams.notificationLevel;
unsigned iteration = 1; unsigned iteration = 1;
unsigned d; unsigned d;
unsigned k; unsigned k;
@@ -716,7 +716,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
data->parameters.splitPoint = splitPoint; data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps; data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict; data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = g_displayLevel; data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel;
/* Check the parameters */ /* Check the parameters */
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
data->ctx->f, accel)) { data->ctx->f, accel)) {
lib/dictBuilder/zdict.c
+16 -9
View File
@@ -139,7 +139,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanForward64( &r, (U64)val ); _BitScanForward64( &r, (U64)val );
return (unsigned)(r>>3); return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) # elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3); return (unsigned)(__builtin_ctzll((U64)val) >> 3);
# else # else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
@@ -150,7 +150,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanForward( &r, (U32)val ); _BitScanForward( &r, (U32)val );
return (unsigned)(r>>3); return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) # elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3); return (unsigned)(__builtin_ctz((U32)val) >> 3);
# else # else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
@@ -163,7 +163,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanReverse64( &r, val ); _BitScanReverse64( &r, val );
return (unsigned)(r>>3); return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) # elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3); return (unsigned)(__builtin_clzll(val) >> 3);
# else # else
unsigned r; unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
@@ -178,7 +178,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanReverse( &r, (unsigned long)val ); _BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3); return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3) # elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3); return (unsigned)(__builtin_clz((U32)val) >> 3);
# else # else
unsigned r; unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
@@ -235,7 +235,7 @@ static dictItem ZDICT_analyzePos(
U32 savings[LLIMIT] = {0}; U32 savings[LLIMIT] = {0};
const BYTE* b = (const BYTE*)buffer; const BYTE* b = (const BYTE*)buffer;
size_t maxLength = LLIMIT; size_t maxLength = LLIMIT;
size_t pos = suffix[start]; size_t pos = (size_t)suffix[start];
U32 end = start; U32 end = start;
dictItem solution; dictItem solution;
@@ -369,7 +369,7 @@ static dictItem ZDICT_analyzePos(
savings[i] = savings[i-1] + (lengthList[i] * (i-3)); savings[i] = savings[i-1] + (lengthList[i] * (i-3));
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength); (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength);
solution.pos = (U32)pos; solution.pos = (U32)pos;
solution.length = (U32)maxLength; solution.length = (U32)maxLength;
@@ -379,7 +379,7 @@ static dictItem ZDICT_analyzePos(
{ U32 id; { U32 id;
for (id=start; id<end; id++) { for (id=start; id<end; id++) {
U32 p, pEnd, length; U32 p, pEnd, length;
U32 const testedPos = suffix[id]; U32 const testedPos = (U32)suffix[id];
if (testedPos == pos) if (testedPos == pos)
length = solution.length; length = solution.length;
else { else {
@@ -442,7 +442,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */ /* append */
int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length);
table[u].savings += elt.length / 8; /* rough approx bonus */ table[u].savings += elt.length / 8; /* rough approx bonus */
if (addedLength > 0) { /* otherwise, elt fully included into existing */ if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength; table[u].length += addedLength;
@@ -766,6 +766,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
pos += fileSizes[u]; pos += fileSizes[u];
} }
if (notificationLevel >= 4) {
/* writeStats */
DISPLAYLEVEL(4, "Offset Code Frequencies : \n");
for (u=0; u<=offcodeMax; u++) {
DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]);
} }
/* analyze, build stats, starting with literals */ /* analyze, build stats, starting with literals */
{ size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
if (HUF_isError(maxNbBits)) { if (HUF_isError(maxNbBits)) {
@@ -872,7 +879,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset);
#else #else
/* at this stage, we don't use the result of "most common first offset", /* at this stage, we don't use the result of "most common first offset",
as the impact of statistics is not properly evaluated */ * as the impact of statistics is not properly evaluated */
MEM_writeLE32(dstPtr+0, repStartValue[0]); MEM_writeLE32(dstPtr+0, repStartValue[0]);
MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]); MEM_writeLE32(dstPtr+8, repStartValue[2]);
programs/benchzstd.c
+18 -25
View File
@@ -70,6 +70,8 @@ static const size_t maxMemory = (sizeof(size_t)==4) ?
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); } #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); }
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
#define OUTPUT(...) { fprintf(stdout, __VA_ARGS__); fflush(NULL); }
#define OUTPUTLEVEL(l, ...) if (displayLevel>=l) { OUTPUT(__VA_ARGS__); }
/* ************************************* /* *************************************
@@ -181,7 +183,7 @@ BMK_initCCtx(ZSTD_CCtx* ctx,
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, (int)comprParams->strategy));
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize)); CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
} }
@@ -371,10 +373,7 @@ BMK_benchMemAdvancedNoAlloc(
if (adv->mode == BMK_decodeOnly) { if (adv->mode == BMK_decodeOnly) {
cSizes[nbBlocks] = thisBlockSize; cSizes[nbBlocks] = thisBlockSize;
benchResult.cSize = thisBlockSize; benchResult.cSize = thisBlockSize;
} } } } }
}
}
}
/* warming up `compressedBuffer` */ /* warming up `compressedBuffer` */
if (adv->mode == BMK_decodeOnly) { if (adv->mode == BMK_decodeOnly) {
@@ -393,8 +392,6 @@ BMK_benchMemAdvancedNoAlloc(
BMK_benchParams_t cbp, dbp; BMK_benchParams_t cbp, dbp;
BMK_initCCtxArgs cctxprep; BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep; BMK_initDCtxArgs dctxprep;
UTIL_HumanReadableSize_t hr_isize;
UTIL_HumanReadableSize_t hr_osize;
cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
cbp.benchPayload = cctx; cbp.benchPayload = cctx;
@@ -431,10 +428,9 @@ BMK_benchMemAdvancedNoAlloc(
dctxprep.dictBuffer = dictBuffer; dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize; dctxprep.dictBufferSize = dictBufferSize;
hr_isize = UTIL_makeHumanReadableSize((U64) srcSize); OUTPUTLEVEL(2, "\r%70s\r", ""); /* blank line */
assert(srcSize < UINT_MAX);
DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ OUTPUTLEVEL(2, "%2s-%-17.17s :%10u -> \r", marks[markNb], displayName, (unsigned)srcSize);
DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix);
while (!(compressionCompleted && decompressionCompleted)) { while (!(compressionCompleted && decompressionCompleted)) {
if (!compressionCompleted) { if (!compressionCompleted) {
@@ -446,7 +442,7 @@ BMK_benchMemAdvancedNoAlloc(
{ BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome); { BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome);
cSize = cResult.sumOfReturn; cSize = cResult.sumOfReturn;
ratio = (double)srcSize / cSize; ratio = (double)srcSize / (double)cSize;
{ BMK_benchResult_t newResult; { BMK_benchResult_t newResult;
newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun); newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
benchResult.cSize = cSize; benchResult.cSize = cSize;
@@ -455,11 +451,10 @@ BMK_benchMemAdvancedNoAlloc(
} } } }
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
hr_osize = UTIL_makeHumanReadableSize((U64) cSize); assert(cSize < UINT_MAX);
DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s\r", OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s\r",
marks[markNb], displayName, marks[markNb], displayName,
hr_isize.precision, hr_isize.value, hr_isize.suffix, (unsigned)srcSize, (unsigned)cSize,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
ratioAccuracy, ratio, ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
} }
@@ -480,11 +475,9 @@ BMK_benchMemAdvancedNoAlloc(
} }
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
hr_osize = UTIL_makeHumanReadableSize((U64) cSize); OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s, %6.1f MB/s \r",
DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s, %6.1f MB/s \r",
marks[markNb], displayName, marks[markNb], displayName,
hr_isize.precision, hr_isize.value, hr_isize.suffix, (unsigned)srcSize, (unsigned)cSize,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
ratioAccuracy, ratio, ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT); (double)benchResult.dSpeed / MB_UNIT);
@@ -543,13 +536,13 @@ BMK_benchMemAdvancedNoAlloc(
double const cSpeed = (double)benchResult.cSpeed / MB_UNIT; double const cSpeed = (double)benchResult.cSpeed / MB_UNIT;
double const dSpeed = (double)benchResult.dSpeed / MB_UNIT; double const dSpeed = (double)benchResult.dSpeed / MB_UNIT;
if (adv->additionalParam) { if (adv->additionalParam) {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
} else { } else {
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
} }
} }
DISPLAYLEVEL(2, "%2i#\n", cLevel); OUTPUTLEVEL(2, "%2i#\n", cLevel);
} /* Bench */ } /* Bench */
benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx);
@@ -678,7 +671,7 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
} }
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
(unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10)); (unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10));
@@ -768,7 +761,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
} }
{ FILE* const f = fopen(fileNamesTable[n], "rb"); { FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); OUTPUTLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]);
programs/util.c
+4 -3
View File
@@ -308,7 +308,8 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf)
return (U64)statbuf->st_size; return (U64)statbuf->st_size;
} }
UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size)
{
UTIL_HumanReadableSize_t hrs; UTIL_HumanReadableSize_t hrs;
if (g_utilDisplayLevel > 3) { if (g_utilDisplayLevel > 3) {
@@ -1121,9 +1122,9 @@ DWORD CountSetBits(ULONG_PTR bitMask)
{ {
DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1;
DWORD bitSetCount = 0; DWORD bitSetCount = 0;
ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT;
DWORD i; DWORD i;
for (i = 0; i <= LSHIFT; ++i) for (i = 0; i <= LSHIFT; ++i)
{ {
bitSetCount += ((bitMask & bitTest)?1:0); bitSetCount += ((bitMask & bitTest)?1:0);
programs/util.h
+6 -2
View File
@@ -180,9 +180,13 @@ U64 UTIL_getFileSize(const char* infilename);
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
/** /**
* Take a size in bytes and prepare the components to pretty-print it in a * Take @size in bytes,
* scaled way. The components in the returned struct should be passed in * prepare the components to pretty-print it in a scaled way.
* The components in the returned struct should be passed in
* precision, value, suffix order to a "%.*f%s" format string. * precision, value, suffix order to a "%.*f%s" format string.
* Output policy is sensible to @g_utilDisplayLevel,
* for verbose mode (@g_utilDisplayLevel >= 4),
* does not scale down.
*/ */
typedef struct { typedef struct {
double value; double value;
tests/automated_benchmarking.py
+5 -5
View File
@@ -87,7 +87,7 @@ def clone_and_build(build):
git clone {github_url} zstd-{user}-{sha} && git clone {github_url} zstd-{user}-{sha} &&
cd zstd-{user}-{sha} && cd zstd-{user}-{sha} &&
{checkout_command} {checkout_command}
make && make -j &&
cd ../ cd ../
""".format( """.format(
user=build["user"], user=build["user"],
@@ -100,7 +100,7 @@ def clone_and_build(build):
) )
return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"]) return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"])
else: else:
os.system("cd ../ && make && cd tests") os.system("cd ../ && make -j && cd tests")
return "../zstd" return "../zstd"
@@ -112,9 +112,9 @@ def parse_benchmark_output(output):
def benchmark_single(executable, level, filename): def benchmark_single(executable, level, filename):
return parse_benchmark_output(( return parse_benchmark_output((
subprocess.run( subprocess.run(
[executable, "-qb{}".format(level), filename], stderr=subprocess.PIPE [executable, "-qb{}".format(level), filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
) )
.stderr.decode("utf-8") .stdout.decode("utf-8")
.split(" ") .split(" ")
)) ))
@@ -145,7 +145,7 @@ def benchmark(build, filenames, levels, iterations):
def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations): def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations):
cspeeds, dspeeds = [], [] cspeeds, dspeeds = [], []
for _ in range(iterations): for _ in range(iterations):
output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stderr=subprocess.PIPE).stderr.decode("utf-8").split(" ") output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stdout=subprocess.PIPE).stdout.decode("utf-8").split(" ")
cspeed, dspeed = parse_benchmark_output(output) cspeed, dspeed = parse_benchmark_output(output)
cspeeds.append(cspeed) cspeeds.append(cspeed)
dspeeds.append(dspeed) dspeeds.append(dspeed)
tests/fuzzer.c
+21 -22
View File
@@ -42,7 +42,7 @@
#include "util.h" #include "util.h"
#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
#include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
@@ -128,7 +128,7 @@ static U32 FUZ_highbit32(U32 v32)
#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; } #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
#define CHECK(fn) { CHECK_NEWV(err, fn); } #define CHECK(fn) { CHECK_NEWV(__err, fn); }
#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
#define CHECK_OP(op, lhs, rhs) { \ #define CHECK_OP(op, lhs, rhs) { \
@@ -1832,8 +1832,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Simple API skippable frame test */ /* Simple API skippable frame test */
DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
{ { U32 i;
U32 i;
unsigned readMagic; unsigned readMagic;
unsigned long long receivedSize; unsigned long long receivedSize;
size_t skippableSize; size_t skippableSize;
@@ -1841,9 +1840,10 @@ static int basicUnitTests(U32 const seed, double compressibility)
char* const skipBuff = (char*)malloc(skipLen); char* const skipBuff = (char*)malloc(skipLen);
assert(skipBuff != NULL); assert(skipBuff != NULL);
for (i = 0; i < skipLen; i++) for (i = 0; i < skipLen; i++)
skipBuff[i] = (BYTE) ((seed + i) % 256); skipBuff[i] = (char) ((seed + i) % 256);
skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, skippableSize = ZSTD_writeSkippableFrame(
skipBuff, skipLen, seed % 15); compressedBuffer, compressedBufferSize,
skipBuff, skipLen, seed % 15);
CHECK_Z(skippableSize); CHECK_Z(skippableSize);
CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
@@ -1860,8 +1860,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
unsigned readMagic; unsigned readMagic;
unsigned long long receivedSize; unsigned long long receivedSize;
size_t skippableSize; size_t skippableSize;
skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, skippableSize = ZSTD_writeSkippableFrame(
CNBuffer, 0, seed % 15); compressedBuffer, compressedBufferSize,
CNBuffer, 0, seed % 15);
CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
@@ -1955,6 +1956,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
} } } }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
/* Note : these tests should be replaced by proper regression tests,
* but existing ones do not focus on small data + dictionary + all levels.
*/
if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
size_t const flatdictSize = 22 KB; size_t const flatdictSize = 22 KB;
size_t const contentSize = 9 KB; size_t const contentSize = 9 KB;
@@ -1963,14 +1967,14 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* These upper bounds are generally within a few bytes of the compressed size */ /* These upper bounds are generally within a few bytes of the compressed size */
size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
3770, 3770, 3770, 3750, 3750, 3770, 3770, 3770, 3750, 3750,
3742, 3670, 3670, 3660, 3660, 3742, 3675, 3674, 3665, 3664,
3660, 3660, 3660, 3660, 3660, 3663, 3662, 3661, 3660, 3660,
3660, 3660, 3660 }; 3660, 3660, 3660 };
size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2890, 2820, 2940, size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2890, 2820, 2940,
2950, 2950, 2925, 2900, 2891, 2950, 2950, 2925, 2900, 2891,
2910, 2910, 2910, 2770, 2760, 2910, 2910, 2910, 2780, 2775,
2750, 2750, 2750, 2750, 2750, 2765, 2760, 2755, 2754, 2753,
2750, 2750, 2750 }; 2753, 2753, 2753 };
int l = 1; int l = 1;
int const maxLevel = ZSTD_maxCLevel(); int const maxLevel = ZSTD_maxCLevel();
/* clevels with strategies that support rowhash on small inputs */ /* clevels with strategies that support rowhash on small inputs */
@@ -3471,11 +3475,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n", DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel); level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
goto _output_error; goto _output_error;
} } } } } }
}
}
}
}
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
@@ -3591,8 +3591,7 @@ static int longUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
{ { size_t cSizeLdm;
size_t cSizeLdm;
size_t cSizeNoLdm; size_t cSizeNoLdm;
ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -3670,7 +3669,7 @@ static int longUnitTests(U32 const seed, double compressibility)
CHECK(cdict != NULL); CHECK(cdict != NULL);
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, attachPref)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize); CHECK_Z(cSize);
tests/regression/results.csv
+117 -117
View File
@@ -11,10 +11,10 @@ silesia.tar, level 6, compress
silesia.tar, level 7, compress simple, 4576829 silesia.tar, level 7, compress simple, 4576829
silesia.tar, level 9, compress simple, 4552584 silesia.tar, level 9, compress simple, 4552584
silesia.tar, level 13, compress simple, 4502956 silesia.tar, level 13, compress simple, 4502956
silesia.tar, level 16, compress simple, 4356834 silesia.tar, level 16, compress simple, 4360527
silesia.tar, level 19, compress simple, 4264388 silesia.tar, level 19, compress simple, 4267266
silesia.tar, uncompressed literals, compress simple, 4861424 silesia.tar, uncompressed literals, compress simple, 4861424
silesia.tar, uncompressed literals optimal, compress simple, 4264388 silesia.tar, uncompressed literals optimal, compress simple, 4267266
silesia.tar, huffman literals, compress simple, 6182241 silesia.tar, huffman literals, compress simple, 6182241
github.tar, level -5, compress simple, 66914 github.tar, level -5, compress simple, 66914
github.tar, level -3, compress simple, 52127 github.tar, level -3, compress simple, 52127
@@ -28,10 +28,10 @@ github.tar, level 6, compress
github.tar, level 7, compress simple, 38073 github.tar, level 7, compress simple, 38073
github.tar, level 9, compress simple, 36767 github.tar, level 9, compress simple, 36767
github.tar, level 13, compress simple, 35501 github.tar, level 13, compress simple, 35501
github.tar, level 16, compress simple, 40255 github.tar, level 16, compress simple, 40471
github.tar, level 19, compress simple, 32837 github.tar, level 19, compress simple, 32134
github.tar, uncompressed literals, compress simple, 38441 github.tar, uncompressed literals, compress simple, 38441
github.tar, uncompressed literals optimal, compress simple, 32837 github.tar, uncompressed literals optimal, compress simple, 32134
github.tar, huffman literals, compress simple, 42560 github.tar, huffman literals, compress simple, 42560
silesia, level -5, compress cctx, 7354675 silesia, level -5, compress cctx, 7354675
silesia, level -3, compress cctx, 6902374 silesia, level -3, compress cctx, 6902374
@@ -45,8 +45,8 @@ silesia, level 6, compress
silesia, level 7, compress cctx, 4567204 silesia, level 7, compress cctx, 4567204
silesia, level 9, compress cctx, 4543310 silesia, level 9, compress cctx, 4543310
silesia, level 13, compress cctx, 4493990 silesia, level 13, compress cctx, 4493990
silesia, level 16, compress cctx, 4360251 silesia, level 16, compress cctx, 4359864
silesia, level 19, compress cctx, 4283237 silesia, level 19, compress cctx, 4296880
silesia, long distance mode, compress cctx, 4849553 silesia, long distance mode, compress cctx, 4849553
silesia, multithreaded, compress cctx, 4849553 silesia, multithreaded, compress cctx, 4849553
silesia, multithreaded long distance mode, compress cctx, 4849553 silesia, multithreaded long distance mode, compress cctx, 4849553
@@ -55,7 +55,7 @@ silesia, small hash log, compress
silesia, small chain log, compress cctx, 4912197 silesia, small chain log, compress cctx, 4912197
silesia, explicit params, compress cctx, 4794481 silesia, explicit params, compress cctx, 4794481
silesia, uncompressed literals, compress cctx, 4849553 silesia, uncompressed literals, compress cctx, 4849553
silesia, uncompressed literals optimal, compress cctx, 4283237 silesia, uncompressed literals optimal, compress cctx, 4296880
silesia, huffman literals, compress cctx, 6177565 silesia, huffman literals, compress cctx, 6177565
silesia, multithreaded with advanced params, compress cctx, 4849553 silesia, multithreaded with advanced params, compress cctx, 4849553
github, level -5, compress cctx, 232315 github, level -5, compress cctx, 232315
@@ -109,8 +109,8 @@ silesia, level 6, zstdcli,
silesia, level 7, zstdcli, 4567252 silesia, level 7, zstdcli, 4567252
silesia, level 9, zstdcli, 4543358 silesia, level 9, zstdcli, 4543358
silesia, level 13, zstdcli, 4494038 silesia, level 13, zstdcli, 4494038
silesia, level 16, zstdcli, 4360299 silesia, level 16, zstdcli, 4359912
silesia, level 19, zstdcli, 4283285 silesia, level 19, zstdcli, 4296928
silesia, long distance mode, zstdcli, 4840807 silesia, long distance mode, zstdcli, 4840807
silesia, multithreaded, zstdcli, 4849601 silesia, multithreaded, zstdcli, 4849601
silesia, multithreaded long distance mode, zstdcli, 4840807 silesia, multithreaded long distance mode, zstdcli, 4840807
@@ -119,7 +119,7 @@ silesia, small hash log, zstdcli,
silesia, small chain log, zstdcli, 4912245 silesia, small chain log, zstdcli, 4912245
silesia, explicit params, zstdcli, 4795856 silesia, explicit params, zstdcli, 4795856
silesia, uncompressed literals, zstdcli, 5128030 silesia, uncompressed literals, zstdcli, 5128030
silesia, uncompressed literals optimal, zstdcli, 4317944 silesia, uncompressed literals optimal, zstdcli, 4319566
silesia, huffman literals, zstdcli, 5326394 silesia, huffman literals, zstdcli, 5326394
silesia, multithreaded with advanced params, zstdcli, 5128030 silesia, multithreaded with advanced params, zstdcli, 5128030
silesia.tar, level -5, zstdcli, 7363866 silesia.tar, level -5, zstdcli, 7363866
@@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli,
silesia.tar, level 7, zstdcli, 4578884 silesia.tar, level 7, zstdcli, 4578884
silesia.tar, level 9, zstdcli, 4553499 silesia.tar, level 9, zstdcli, 4553499
silesia.tar, level 13, zstdcli, 4502960 silesia.tar, level 13, zstdcli, 4502960
silesia.tar, level 16, zstdcli, 4356838 silesia.tar, level 16, zstdcli, 4360531
silesia.tar, level 19, zstdcli, 4264392 silesia.tar, level 19, zstdcli, 4267270
silesia.tar, no source size, zstdcli, 4861508 silesia.tar, no source size, zstdcli, 4861508
silesia.tar, long distance mode, zstdcli, 4853225 silesia.tar, long distance mode, zstdcli, 4853225
silesia.tar, multithreaded, zstdcli, 4861512 silesia.tar, multithreaded, zstdcli, 4861512
@@ -145,7 +145,7 @@ silesia.tar, small hash log, zstdcli,
silesia.tar, small chain log, zstdcli, 4917022 silesia.tar, small chain log, zstdcli, 4917022
silesia.tar, explicit params, zstdcli, 4821274 silesia.tar, explicit params, zstdcli, 4821274
silesia.tar, uncompressed literals, zstdcli, 5129559 silesia.tar, uncompressed literals, zstdcli, 5129559
silesia.tar, uncompressed literals optimal, zstdcli, 4307404 silesia.tar, uncompressed literals optimal, zstdcli, 4310145
silesia.tar, huffman literals, zstdcli, 5344915 silesia.tar, huffman literals, zstdcli, 5344915
silesia.tar, multithreaded with advanced params, zstdcli, 5129559 silesia.tar, multithreaded with advanced params, zstdcli, 5129559
github, level -5, zstdcli, 234315 github, level -5, zstdcli, 234315
@@ -210,11 +210,11 @@ github.tar, level 7 with dict, zstdcli,
github.tar, level 9, zstdcli, 36771 github.tar, level 9, zstdcli, 36771
github.tar, level 9 with dict, zstdcli, 36623 github.tar, level 9 with dict, zstdcli, 36623
github.tar, level 13, zstdcli, 35505 github.tar, level 13, zstdcli, 35505
github.tar, level 13 with dict, zstdcli, 38730 github.tar, level 13 with dict, zstdcli, 37134
github.tar, level 16, zstdcli, 40259 github.tar, level 16, zstdcli, 40475
github.tar, level 16 with dict, zstdcli, 33643 github.tar, level 16 with dict, zstdcli, 33382
github.tar, level 19, zstdcli, 32841 github.tar, level 19, zstdcli, 32138
github.tar, level 19 with dict, zstdcli, 32899 github.tar, level 19 with dict, zstdcli, 32713
github.tar, no source size, zstdcli, 38442 github.tar, no source size, zstdcli, 38442
github.tar, no source size with dict, zstdcli, 38004 github.tar, no source size with dict, zstdcli, 38004
github.tar, long distance mode, zstdcli, 39730 github.tar, long distance mode, zstdcli, 39730
@@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli,
github.tar, small chain log, zstdcli, 41673 github.tar, small chain log, zstdcli, 41673
github.tar, explicit params, zstdcli, 41227 github.tar, explicit params, zstdcli, 41227
github.tar, uncompressed literals, zstdcli, 41126 github.tar, uncompressed literals, zstdcli, 41126
github.tar, uncompressed literals optimal, zstdcli, 35392 github.tar, uncompressed literals optimal, zstdcli, 35401
github.tar, huffman literals, zstdcli, 38857 github.tar, huffman literals, zstdcli, 38857
github.tar, multithreaded with advanced params, zstdcli, 41126 github.tar, multithreaded with advanced params, zstdcli, 41126
silesia, level -5, advanced one pass, 7354675 silesia, level -5, advanced one pass, 7354675
@@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced
silesia, level 12 row 1, advanced one pass, 4503116 silesia, level 12 row 1, advanced one pass, 4503116
silesia, level 12 row 2, advanced one pass, 4505153 silesia, level 12 row 2, advanced one pass, 4505153
silesia, level 13, advanced one pass, 4493990 silesia, level 13, advanced one pass, 4493990
silesia, level 16, advanced one pass, 4360251 silesia, level 16, advanced one pass, 4359864
silesia, level 19, advanced one pass, 4283237 silesia, level 19, advanced one pass, 4296880
silesia, no source size, advanced one pass, 4849553 silesia, no source size, advanced one pass, 4849553
silesia, long distance mode, advanced one pass, 4840737 silesia, long distance mode, advanced one pass, 4840737
silesia, multithreaded, advanced one pass, 4849553 silesia, multithreaded, advanced one pass, 4849553
@@ -259,7 +259,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced one pass, 4912197 silesia, small chain log, advanced one pass, 4912197
silesia, explicit params, advanced one pass, 4795856 silesia, explicit params, advanced one pass, 4795856
silesia, uncompressed literals, advanced one pass, 5127982 silesia, uncompressed literals, advanced one pass, 5127982
silesia, uncompressed literals optimal, advanced one pass, 4317896 silesia, uncompressed literals optimal, advanced one pass, 4319518
silesia, huffman literals, advanced one pass, 5326346 silesia, huffman literals, advanced one pass, 5326346
silesia, multithreaded with advanced params, advanced one pass, 5127982 silesia, multithreaded with advanced params, advanced one pass, 5127982
silesia.tar, level -5, advanced one pass, 7359401 silesia.tar, level -5, advanced one pass, 7359401
@@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced
silesia.tar, level 12 row 1, advanced one pass, 4513604 silesia.tar, level 12 row 1, advanced one pass, 4513604
silesia.tar, level 12 row 2, advanced one pass, 4514568 silesia.tar, level 12 row 2, advanced one pass, 4514568
silesia.tar, level 13, advanced one pass, 4502956 silesia.tar, level 13, advanced one pass, 4502956
silesia.tar, level 16, advanced one pass, 4356834 silesia.tar, level 16, advanced one pass, 4360527
silesia.tar, level 19, advanced one pass, 4264388 silesia.tar, level 19, advanced one pass, 4267266
silesia.tar, no source size, advanced one pass, 4861424 silesia.tar, no source size, advanced one pass, 4861424
silesia.tar, long distance mode, advanced one pass, 4847752 silesia.tar, long distance mode, advanced one pass, 4847752
silesia.tar, multithreaded, advanced one pass, 4861508 silesia.tar, multithreaded, advanced one pass, 4861508
@@ -293,7 +293,7 @@ silesia.tar, small hash log, advanced
silesia.tar, small chain log, advanced one pass, 4917041 silesia.tar, small chain log, advanced one pass, 4917041
silesia.tar, explicit params, advanced one pass, 4807381 silesia.tar, explicit params, advanced one pass, 4807381
silesia.tar, uncompressed literals, advanced one pass, 5129458 silesia.tar, uncompressed literals, advanced one pass, 5129458
silesia.tar, uncompressed literals optimal, advanced one pass, 4307400 silesia.tar, uncompressed literals optimal, advanced one pass, 4310141
silesia.tar, huffman literals, advanced one pass, 5344545 silesia.tar, huffman literals, advanced one pass, 5344545
silesia.tar, multithreaded with advanced params, advanced one pass, 5129555 silesia.tar, multithreaded with advanced params, advanced one pass, 5129555
github, level -5, advanced one pass, 232315 github, level -5, advanced one pass, 232315
@@ -516,23 +516,23 @@ github.tar, level 12 row 2 with dict dds, advanced
github.tar, level 12 row 2 with dict copy, advanced one pass, 36609 github.tar, level 12 row 2 with dict copy, advanced one pass, 36609
github.tar, level 12 row 2 with dict load, advanced one pass, 36460 github.tar, level 12 row 2 with dict load, advanced one pass, 36460
github.tar, level 13, advanced one pass, 35501 github.tar, level 13, advanced one pass, 35501
github.tar, level 13 with dict, advanced one pass, 38726 github.tar, level 13 with dict, advanced one pass, 37130
github.tar, level 13 with dict dms, advanced one pass, 38903 github.tar, level 13 with dict dms, advanced one pass, 37267
github.tar, level 13 with dict dds, advanced one pass, 38903 github.tar, level 13 with dict dds, advanced one pass, 37267
github.tar, level 13 with dict copy, advanced one pass, 38726 github.tar, level 13 with dict copy, advanced one pass, 37130
github.tar, level 13 with dict load, advanced one pass, 36010 github.tar, level 13 with dict load, advanced one pass, 36010
github.tar, level 16, advanced one pass, 40255 github.tar, level 16, advanced one pass, 40471
github.tar, level 16 with dict, advanced one pass, 33639 github.tar, level 16 with dict, advanced one pass, 33378
github.tar, level 16 with dict dms, advanced one pass, 33544 github.tar, level 16 with dict dms, advanced one pass, 33213
github.tar, level 16 with dict dds, advanced one pass, 33544 github.tar, level 16 with dict dds, advanced one pass, 33213
github.tar, level 16 with dict copy, advanced one pass, 33639 github.tar, level 16 with dict copy, advanced one pass, 33378
github.tar, level 16 with dict load, advanced one pass, 39353 github.tar, level 16 with dict load, advanced one pass, 39081
github.tar, level 19, advanced one pass, 32837 github.tar, level 19, advanced one pass, 32134
github.tar, level 19 with dict, advanced one pass, 32895 github.tar, level 19 with dict, advanced one pass, 32709
github.tar, level 19 with dict dms, advanced one pass, 32672 github.tar, level 19 with dict dms, advanced one pass, 32553
github.tar, level 19 with dict dds, advanced one pass, 32672 github.tar, level 19 with dict dds, advanced one pass, 32553
github.tar, level 19 with dict copy, advanced one pass, 32895 github.tar, level 19 with dict copy, advanced one pass, 32709
github.tar, level 19 with dict load, advanced one pass, 32676 github.tar, level 19 with dict load, advanced one pass, 32474
github.tar, no source size, advanced one pass, 38441 github.tar, no source size, advanced one pass, 38441
github.tar, no source size with dict, advanced one pass, 37995 github.tar, no source size with dict, advanced one pass, 37995
github.tar, long distance mode, advanced one pass, 39757 github.tar, long distance mode, advanced one pass, 39757
@@ -543,7 +543,7 @@ github.tar, small hash log, advanced
github.tar, small chain log, advanced one pass, 41669 github.tar, small chain log, advanced one pass, 41669
github.tar, explicit params, advanced one pass, 41227 github.tar, explicit params, advanced one pass, 41227
github.tar, uncompressed literals, advanced one pass, 41122 github.tar, uncompressed literals, advanced one pass, 41122
github.tar, uncompressed literals optimal, advanced one pass, 35388 github.tar, uncompressed literals optimal, advanced one pass, 35397
github.tar, huffman literals, advanced one pass, 38853 github.tar, huffman literals, advanced one pass, 38853
github.tar, multithreaded with advanced params, advanced one pass, 41122 github.tar, multithreaded with advanced params, advanced one pass, 41122
silesia, level -5, advanced one pass small out, 7354675 silesia, level -5, advanced one pass small out, 7354675
@@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced
silesia, level 12 row 1, advanced one pass small out, 4503116 silesia, level 12 row 1, advanced one pass small out, 4503116
silesia, level 12 row 2, advanced one pass small out, 4505153 silesia, level 12 row 2, advanced one pass small out, 4505153
silesia, level 13, advanced one pass small out, 4493990 silesia, level 13, advanced one pass small out, 4493990
silesia, level 16, advanced one pass small out, 4360251 silesia, level 16, advanced one pass small out, 4359864
silesia, level 19, advanced one pass small out, 4283237 silesia, level 19, advanced one pass small out, 4296880
silesia, no source size, advanced one pass small out, 4849553 silesia, no source size, advanced one pass small out, 4849553
silesia, long distance mode, advanced one pass small out, 4840737 silesia, long distance mode, advanced one pass small out, 4840737
silesia, multithreaded, advanced one pass small out, 4849553 silesia, multithreaded, advanced one pass small out, 4849553
@@ -577,7 +577,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced one pass small out, 4912197 silesia, small chain log, advanced one pass small out, 4912197
silesia, explicit params, advanced one pass small out, 4795856 silesia, explicit params, advanced one pass small out, 4795856
silesia, uncompressed literals, advanced one pass small out, 5127982 silesia, uncompressed literals, advanced one pass small out, 5127982
silesia, uncompressed literals optimal, advanced one pass small out, 4317896 silesia, uncompressed literals optimal, advanced one pass small out, 4319518
silesia, huffman literals, advanced one pass small out, 5326346 silesia, huffman literals, advanced one pass small out, 5326346
silesia, multithreaded with advanced params, advanced one pass small out, 5127982 silesia, multithreaded with advanced params, advanced one pass small out, 5127982
silesia.tar, level -5, advanced one pass small out, 7359401 silesia.tar, level -5, advanced one pass small out, 7359401
@@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced
silesia.tar, level 12 row 1, advanced one pass small out, 4513604 silesia.tar, level 12 row 1, advanced one pass small out, 4513604
silesia.tar, level 12 row 2, advanced one pass small out, 4514568 silesia.tar, level 12 row 2, advanced one pass small out, 4514568
silesia.tar, level 13, advanced one pass small out, 4502956 silesia.tar, level 13, advanced one pass small out, 4502956
silesia.tar, level 16, advanced one pass small out, 4356834 silesia.tar, level 16, advanced one pass small out, 4360527
silesia.tar, level 19, advanced one pass small out, 4264388 silesia.tar, level 19, advanced one pass small out, 4267266
silesia.tar, no source size, advanced one pass small out, 4861424 silesia.tar, no source size, advanced one pass small out, 4861424
silesia.tar, long distance mode, advanced one pass small out, 4847752 silesia.tar, long distance mode, advanced one pass small out, 4847752
silesia.tar, multithreaded, advanced one pass small out, 4861508 silesia.tar, multithreaded, advanced one pass small out, 4861508
@@ -611,7 +611,7 @@ silesia.tar, small hash log, advanced
silesia.tar, small chain log, advanced one pass small out, 4917041 silesia.tar, small chain log, advanced one pass small out, 4917041
silesia.tar, explicit params, advanced one pass small out, 4807381 silesia.tar, explicit params, advanced one pass small out, 4807381
silesia.tar, uncompressed literals, advanced one pass small out, 5129458 silesia.tar, uncompressed literals, advanced one pass small out, 5129458
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4307400 silesia.tar, uncompressed literals optimal, advanced one pass small out, 4310141
silesia.tar, huffman literals, advanced one pass small out, 5344545 silesia.tar, huffman literals, advanced one pass small out, 5344545
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555
github, level -5, advanced one pass small out, 232315 github, level -5, advanced one pass small out, 232315
@@ -834,23 +834,23 @@ github.tar, level 12 row 2 with dict dds, advanced
github.tar, level 12 row 2 with dict copy, advanced one pass small out, 36609 github.tar, level 12 row 2 with dict copy, advanced one pass small out, 36609
github.tar, level 12 row 2 with dict load, advanced one pass small out, 36460 github.tar, level 12 row 2 with dict load, advanced one pass small out, 36460
github.tar, level 13, advanced one pass small out, 35501 github.tar, level 13, advanced one pass small out, 35501
github.tar, level 13 with dict, advanced one pass small out, 38726 github.tar, level 13 with dict, advanced one pass small out, 37130
github.tar, level 13 with dict dms, advanced one pass small out, 38903 github.tar, level 13 with dict dms, advanced one pass small out, 37267
github.tar, level 13 with dict dds, advanced one pass small out, 38903 github.tar, level 13 with dict dds, advanced one pass small out, 37267
github.tar, level 13 with dict copy, advanced one pass small out, 38726 github.tar, level 13 with dict copy, advanced one pass small out, 37130
github.tar, level 13 with dict load, advanced one pass small out, 36010 github.tar, level 13 with dict load, advanced one pass small out, 36010
github.tar, level 16, advanced one pass small out, 40255 github.tar, level 16, advanced one pass small out, 40471
github.tar, level 16 with dict, advanced one pass small out, 33639 github.tar, level 16 with dict, advanced one pass small out, 33378
github.tar, level 16 with dict dms, advanced one pass small out, 33544 github.tar, level 16 with dict dms, advanced one pass small out, 33213
github.tar, level 16 with dict dds, advanced one pass small out, 33544 github.tar, level 16 with dict dds, advanced one pass small out, 33213
github.tar, level 16 with dict copy, advanced one pass small out, 33639 github.tar, level 16 with dict copy, advanced one pass small out, 33378
github.tar, level 16 with dict load, advanced one pass small out, 39353 github.tar, level 16 with dict load, advanced one pass small out, 39081
github.tar, level 19, advanced one pass small out, 32837 github.tar, level 19, advanced one pass small out, 32134
github.tar, level 19 with dict, advanced one pass small out, 32895 github.tar, level 19 with dict, advanced one pass small out, 32709
github.tar, level 19 with dict dms, advanced one pass small out, 32672 github.tar, level 19 with dict dms, advanced one pass small out, 32553
github.tar, level 19 with dict dds, advanced one pass small out, 32672 github.tar, level 19 with dict dds, advanced one pass small out, 32553
github.tar, level 19 with dict copy, advanced one pass small out, 32895 github.tar, level 19 with dict copy, advanced one pass small out, 32709
github.tar, level 19 with dict load, advanced one pass small out, 32676 github.tar, level 19 with dict load, advanced one pass small out, 32474
github.tar, no source size, advanced one pass small out, 38441 github.tar, no source size, advanced one pass small out, 38441
github.tar, no source size with dict, advanced one pass small out, 37995 github.tar, no source size with dict, advanced one pass small out, 37995
github.tar, long distance mode, advanced one pass small out, 39757 github.tar, long distance mode, advanced one pass small out, 39757
@@ -861,7 +861,7 @@ github.tar, small hash log, advanced
github.tar, small chain log, advanced one pass small out, 41669 github.tar, small chain log, advanced one pass small out, 41669
github.tar, explicit params, advanced one pass small out, 41227 github.tar, explicit params, advanced one pass small out, 41227
github.tar, uncompressed literals, advanced one pass small out, 41122 github.tar, uncompressed literals, advanced one pass small out, 41122
github.tar, uncompressed literals optimal, advanced one pass small out, 35388 github.tar, uncompressed literals optimal, advanced one pass small out, 35397
github.tar, huffman literals, advanced one pass small out, 38853 github.tar, huffman literals, advanced one pass small out, 38853
github.tar, multithreaded with advanced params, advanced one pass small out, 41122 github.tar, multithreaded with advanced params, advanced one pass small out, 41122
silesia, level -5, advanced streaming, 7292053 silesia, level -5, advanced streaming, 7292053
@@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced
silesia, level 12 row 1, advanced streaming, 4503116 silesia, level 12 row 1, advanced streaming, 4503116
silesia, level 12 row 2, advanced streaming, 4505153 silesia, level 12 row 2, advanced streaming, 4505153
silesia, level 13, advanced streaming, 4493990 silesia, level 13, advanced streaming, 4493990
silesia, level 16, advanced streaming, 4360251 silesia, level 16, advanced streaming, 4359864
silesia, level 19, advanced streaming, 4283237 silesia, level 19, advanced streaming, 4296880
silesia, no source size, advanced streaming, 4849517 silesia, no source size, advanced streaming, 4849517
silesia, long distance mode, advanced streaming, 4840737 silesia, long distance mode, advanced streaming, 4840737
silesia, multithreaded, advanced streaming, 4849553 silesia, multithreaded, advanced streaming, 4849553
@@ -895,7 +895,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced streaming, 4912197 silesia, small chain log, advanced streaming, 4912197
silesia, explicit params, advanced streaming, 4795883 silesia, explicit params, advanced streaming, 4795883
silesia, uncompressed literals, advanced streaming, 5127982 silesia, uncompressed literals, advanced streaming, 5127982
silesia, uncompressed literals optimal, advanced streaming, 4317896 silesia, uncompressed literals optimal, advanced streaming, 4319518
silesia, huffman literals, advanced streaming, 5332234 silesia, huffman literals, advanced streaming, 5332234
silesia, multithreaded with advanced params, advanced streaming, 5127982 silesia, multithreaded with advanced params, advanced streaming, 5127982
silesia.tar, level -5, advanced streaming, 7260007 silesia.tar, level -5, advanced streaming, 7260007
@@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced
silesia.tar, level 12 row 1, advanced streaming, 4513604 silesia.tar, level 12 row 1, advanced streaming, 4513604
silesia.tar, level 12 row 2, advanced streaming, 4514569 silesia.tar, level 12 row 2, advanced streaming, 4514569
silesia.tar, level 13, advanced streaming, 4502956 silesia.tar, level 13, advanced streaming, 4502956
silesia.tar, level 16, advanced streaming, 4356834 silesia.tar, level 16, advanced streaming, 4360527
silesia.tar, level 19, advanced streaming, 4264388 silesia.tar, level 19, advanced streaming, 4267266
silesia.tar, no source size, advanced streaming, 4861422 silesia.tar, no source size, advanced streaming, 4861422
silesia.tar, long distance mode, advanced streaming, 4847752 silesia.tar, long distance mode, advanced streaming, 4847752
silesia.tar, multithreaded, advanced streaming, 4861508 silesia.tar, multithreaded, advanced streaming, 4861508
@@ -929,7 +929,7 @@ silesia.tar, small hash log, advanced
silesia.tar, small chain log, advanced streaming, 4917021 silesia.tar, small chain log, advanced streaming, 4917021
silesia.tar, explicit params, advanced streaming, 4807401 silesia.tar, explicit params, advanced streaming, 4807401
silesia.tar, uncompressed literals, advanced streaming, 5129461 silesia.tar, uncompressed literals, advanced streaming, 5129461
silesia.tar, uncompressed literals optimal, advanced streaming, 4307400 silesia.tar, uncompressed literals optimal, advanced streaming, 4310141
silesia.tar, huffman literals, advanced streaming, 5350519 silesia.tar, huffman literals, advanced streaming, 5350519
silesia.tar, multithreaded with advanced params, advanced streaming, 5129555 silesia.tar, multithreaded with advanced params, advanced streaming, 5129555
github, level -5, advanced streaming, 232315 github, level -5, advanced streaming, 232315
@@ -1152,23 +1152,23 @@ github.tar, level 12 row 2 with dict dds, advanced
github.tar, level 12 row 2 with dict copy, advanced streaming, 36609 github.tar, level 12 row 2 with dict copy, advanced streaming, 36609
github.tar, level 12 row 2 with dict load, advanced streaming, 36460 github.tar, level 12 row 2 with dict load, advanced streaming, 36460
github.tar, level 13, advanced streaming, 35501 github.tar, level 13, advanced streaming, 35501
github.tar, level 13 with dict, advanced streaming, 38726 github.tar, level 13 with dict, advanced streaming, 37130
github.tar, level 13 with dict dms, advanced streaming, 38903 github.tar, level 13 with dict dms, advanced streaming, 37267
github.tar, level 13 with dict dds, advanced streaming, 38903 github.tar, level 13 with dict dds, advanced streaming, 37267
github.tar, level 13 with dict copy, advanced streaming, 38726 github.tar, level 13 with dict copy, advanced streaming, 37130
github.tar, level 13 with dict load, advanced streaming, 36010 github.tar, level 13 with dict load, advanced streaming, 36010
github.tar, level 16, advanced streaming, 40255 github.tar, level 16, advanced streaming, 40471
github.tar, level 16 with dict, advanced streaming, 33639 github.tar, level 16 with dict, advanced streaming, 33378
github.tar, level 16 with dict dms, advanced streaming, 33544 github.tar, level 16 with dict dms, advanced streaming, 33213
github.tar, level 16 with dict dds, advanced streaming, 33544 github.tar, level 16 with dict dds, advanced streaming, 33213
github.tar, level 16 with dict copy, advanced streaming, 33639 github.tar, level 16 with dict copy, advanced streaming, 33378
github.tar, level 16 with dict load, advanced streaming, 39353 github.tar, level 16 with dict load, advanced streaming, 39081
github.tar, level 19, advanced streaming, 32837 github.tar, level 19, advanced streaming, 32134
github.tar, level 19 with dict, advanced streaming, 32895 github.tar, level 19 with dict, advanced streaming, 32709
github.tar, level 19 with dict dms, advanced streaming, 32672 github.tar, level 19 with dict dms, advanced streaming, 32553
github.tar, level 19 with dict dds, advanced streaming, 32672 github.tar, level 19 with dict dds, advanced streaming, 32553
github.tar, level 19 with dict copy, advanced streaming, 32895 github.tar, level 19 with dict copy, advanced streaming, 32709
github.tar, level 19 with dict load, advanced streaming, 32676 github.tar, level 19 with dict load, advanced streaming, 32474
github.tar, no source size, advanced streaming, 38438 github.tar, no source size, advanced streaming, 38438
github.tar, no source size with dict, advanced streaming, 38000 github.tar, no source size with dict, advanced streaming, 38000
github.tar, long distance mode, advanced streaming, 39757 github.tar, long distance mode, advanced streaming, 39757
@@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced
github.tar, small chain log, advanced streaming, 41669 github.tar, small chain log, advanced streaming, 41669
github.tar, explicit params, advanced streaming, 41227 github.tar, explicit params, advanced streaming, 41227
github.tar, uncompressed literals, advanced streaming, 41122 github.tar, uncompressed literals, advanced streaming, 41122
github.tar, uncompressed literals optimal, advanced streaming, 35388 github.tar, uncompressed literals optimal, advanced streaming, 35397
github.tar, huffman literals, advanced streaming, 38874 github.tar, huffman literals, advanced streaming, 38874
github.tar, multithreaded with advanced params, advanced streaming, 41122 github.tar, multithreaded with advanced params, advanced streaming, 41122
silesia, level -5, old streaming, 7292053 silesia, level -5, old streaming, 7292053
@@ -1194,11 +1194,11 @@ silesia, level 6, old stre
silesia, level 7, old streaming, 4567204 silesia, level 7, old streaming, 4567204
silesia, level 9, old streaming, 4543310 silesia, level 9, old streaming, 4543310
silesia, level 13, old streaming, 4493990 silesia, level 13, old streaming, 4493990
silesia, level 16, old streaming, 4360251 silesia, level 16, old streaming, 4359864
silesia, level 19, old streaming, 4283237 silesia, level 19, old streaming, 4296880
silesia, no source size, old streaming, 4849517 silesia, no source size, old streaming, 4849517
silesia, uncompressed literals, old streaming, 4849553 silesia, uncompressed literals, old streaming, 4849553
silesia, uncompressed literals optimal, old streaming, 4283237 silesia, uncompressed literals optimal, old streaming, 4296880
silesia, huffman literals, old streaming, 6183923 silesia, huffman literals, old streaming, 6183923
silesia.tar, level -5, old streaming, 7260007 silesia.tar, level -5, old streaming, 7260007
silesia.tar, level -3, old streaming, 6845151 silesia.tar, level -3, old streaming, 6845151
@@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming, 4576831 silesia.tar, level 7, old streaming, 4576831
silesia.tar, level 9, old streaming, 4552590 silesia.tar, level 9, old streaming, 4552590
silesia.tar, level 13, old streaming, 4502956 silesia.tar, level 13, old streaming, 4502956
silesia.tar, level 16, old streaming, 4356834 silesia.tar, level 16, old streaming, 4360527
silesia.tar, level 19, old streaming, 4264388 silesia.tar, level 19, old streaming, 4267266
silesia.tar, no source size, old streaming, 4861422 silesia.tar, no source size, old streaming, 4861422
silesia.tar, uncompressed literals, old streaming, 4861426 silesia.tar, uncompressed literals, old streaming, 4861426
silesia.tar, uncompressed literals optimal, old streaming, 4264388 silesia.tar, uncompressed literals optimal, old streaming, 4267266
silesia.tar, huffman literals, old streaming, 6187938 silesia.tar, huffman literals, old streaming, 6187938
github, level -5, old streaming, 232315 github, level -5, old streaming, 232315
github, level -5 with dict, old streaming, 46718 github, level -5 with dict, old streaming, 46718
@@ -1274,15 +1274,15 @@ github.tar, level 7 with dict, old stre
github.tar, level 9, old streaming, 36767 github.tar, level 9, old streaming, 36767
github.tar, level 9 with dict, old streaming, 36457 github.tar, level 9 with dict, old streaming, 36457
github.tar, level 13, old streaming, 35501 github.tar, level 13, old streaming, 35501
github.tar, level 13 with dict, old streaming, 38726 github.tar, level 13 with dict, old streaming, 37130
github.tar, level 16, old streaming, 40255 github.tar, level 16, old streaming, 40471
github.tar, level 16 with dict, old streaming, 33639 github.tar, level 16 with dict, old streaming, 33378
github.tar, level 19, old streaming, 32837 github.tar, level 19, old streaming, 32134
github.tar, level 19 with dict, old streaming, 32895 github.tar, level 19 with dict, old streaming, 32709
github.tar, no source size, old streaming, 38438 github.tar, no source size, old streaming, 38438
github.tar, no source size with dict, old streaming, 38000 github.tar, no source size with dict, old streaming, 38000
github.tar, uncompressed literals, old streaming, 38441 github.tar, uncompressed literals, old streaming, 38441
github.tar, uncompressed literals optimal, old streaming, 32837 github.tar, uncompressed literals optimal, old streaming, 32134
github.tar, huffman literals, old streaming, 42536 github.tar, huffman literals, old streaming, 42536
silesia, level -5, old streaming advanced, 7292053 silesia, level -5, old streaming advanced, 7292053
silesia, level -3, old streaming advanced, 6867875 silesia, level -3, old streaming advanced, 6867875
@@ -1296,8 +1296,8 @@ silesia, level 6, old stre
silesia, level 7, old streaming advanced, 4567204 silesia, level 7, old streaming advanced, 4567204
silesia, level 9, old streaming advanced, 4543310 silesia, level 9, old streaming advanced, 4543310
silesia, level 13, old streaming advanced, 4493990 silesia, level 13, old streaming advanced, 4493990
silesia, level 16, old streaming advanced, 4360251 silesia, level 16, old streaming advanced, 4359864
silesia, level 19, old streaming advanced, 4283237 silesia, level 19, old streaming advanced, 4296880
silesia, no source size, old streaming advanced, 4849517 silesia, no source size, old streaming advanced, 4849517
silesia, long distance mode, old streaming advanced, 4849553 silesia, long distance mode, old streaming advanced, 4849553
silesia, multithreaded, old streaming advanced, 4849553 silesia, multithreaded, old streaming advanced, 4849553
@@ -1307,7 +1307,7 @@ silesia, small hash log, old stre
silesia, small chain log, old streaming advanced, 4912197 silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795883 silesia, explicit params, old streaming advanced, 4795883
silesia, uncompressed literals, old streaming advanced, 4849553 silesia, uncompressed literals, old streaming advanced, 4849553
silesia, uncompressed literals optimal, old streaming advanced, 4283237 silesia, uncompressed literals optimal, old streaming advanced, 4296880
silesia, huffman literals, old streaming advanced, 6183923 silesia, huffman literals, old streaming advanced, 6183923
silesia, multithreaded with advanced params, old streaming advanced, 4849553 silesia, multithreaded with advanced params, old streaming advanced, 4849553
silesia.tar, level -5, old streaming advanced, 7260007 silesia.tar, level -5, old streaming advanced, 7260007
@@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming advanced, 4576831 silesia.tar, level 7, old streaming advanced, 4576831
silesia.tar, level 9, old streaming advanced, 4552590 silesia.tar, level 9, old streaming advanced, 4552590
silesia.tar, level 13, old streaming advanced, 4502956 silesia.tar, level 13, old streaming advanced, 4502956
silesia.tar, level 16, old streaming advanced, 4356834 silesia.tar, level 16, old streaming advanced, 4360527
silesia.tar, level 19, old streaming advanced, 4264388 silesia.tar, level 19, old streaming advanced, 4267266
silesia.tar, no source size, old streaming advanced, 4861422 silesia.tar, no source size, old streaming advanced, 4861422
silesia.tar, long distance mode, old streaming advanced, 4861426 silesia.tar, long distance mode, old streaming advanced, 4861426
silesia.tar, multithreaded, old streaming advanced, 4861426 silesia.tar, multithreaded, old streaming advanced, 4861426
@@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre
silesia.tar, small chain log, old streaming advanced, 4917021 silesia.tar, small chain log, old streaming advanced, 4917021
silesia.tar, explicit params, old streaming advanced, 4807401 silesia.tar, explicit params, old streaming advanced, 4807401
silesia.tar, uncompressed literals, old streaming advanced, 4861426 silesia.tar, uncompressed literals, old streaming advanced, 4861426
silesia.tar, uncompressed literals optimal, old streaming advanced, 4264388 silesia.tar, uncompressed literals optimal, old streaming advanced, 4267266
silesia.tar, huffman literals, old streaming advanced, 6187938 silesia.tar, huffman literals, old streaming advanced, 6187938
silesia.tar, multithreaded with advanced params, old streaming advanced, 4861426 silesia.tar, multithreaded with advanced params, old streaming advanced, 4861426
github, level -5, old streaming advanced, 241214 github, level -5, old streaming advanced, 241214
@@ -1401,10 +1401,10 @@ github.tar, level 9, old stre
github.tar, level 9 with dict, old streaming advanced, 36233 github.tar, level 9 with dict, old streaming advanced, 36233
github.tar, level 13, old streaming advanced, 35501 github.tar, level 13, old streaming advanced, 35501
github.tar, level 13 with dict, old streaming advanced, 35807 github.tar, level 13 with dict, old streaming advanced, 35807
github.tar, level 16, old streaming advanced, 40255 github.tar, level 16, old streaming advanced, 40471
github.tar, level 16 with dict, old streaming advanced, 38736 github.tar, level 16 with dict, old streaming advanced, 38578
github.tar, level 19, old streaming advanced, 32837 github.tar, level 19, old streaming advanced, 32134
github.tar, level 19 with dict, old streaming advanced, 32876 github.tar, level 19 with dict, old streaming advanced, 32702
github.tar, no source size, old streaming advanced, 38438 github.tar, no source size, old streaming advanced, 38438
github.tar, no source size with dict, old streaming advanced, 38015 github.tar, no source size with dict, old streaming advanced, 38015
github.tar, long distance mode, old streaming advanced, 38441 github.tar, long distance mode, old streaming advanced, 38441
@@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre
github.tar, small chain log, old streaming advanced, 41669 github.tar, small chain log, old streaming advanced, 41669
github.tar, explicit params, old streaming advanced, 41227 github.tar, explicit params, old streaming advanced, 41227
github.tar, uncompressed literals, old streaming advanced, 38441 github.tar, uncompressed literals, old streaming advanced, 38441
github.tar, uncompressed literals optimal, old streaming advanced, 32837 github.tar, uncompressed literals optimal, old streaming advanced, 32134
github.tar, huffman literals, old streaming advanced, 42536 github.tar, huffman literals, old streaming advanced, 42536
github.tar, multithreaded with advanced params, old streaming advanced, 38441 github.tar, multithreaded with advanced params, old streaming advanced, 38441
github, level -5 with dict, old streaming cdict, 46718 github, level -5 with dict, old streaming cdict, 46718
@@ -1445,8 +1445,8 @@ github.tar, level 6 with dict, old stre
github.tar, level 7 with dict, old streaming cdict, 37371 github.tar, level 7 with dict, old streaming cdict, 37371
github.tar, level 9 with dict, old streaming cdict, 36352 github.tar, level 9 with dict, old streaming cdict, 36352
github.tar, level 13 with dict, old streaming cdict, 36010 github.tar, level 13 with dict, old streaming cdict, 36010
github.tar, level 16 with dict, old streaming cdict, 39353 github.tar, level 16 with dict, old streaming cdict, 39081
github.tar, level 19 with dict, old streaming cdict, 32676 github.tar, level 19 with dict, old streaming cdict, 32474
github.tar, no source size with dict, old streaming cdict, 38000 github.tar, no source size with dict, old streaming cdict, 38000
github, level -5 with dict, old streaming advanced cdict, 49562 github, level -5 with dict, old streaming advanced cdict, 49562
github, level -3 with dict, old streaming advanced cdict, 44956 github, level -3 with dict, old streaming advanced cdict, 44956
@@ -1475,6 +1475,6 @@ github.tar, level 6 with dict, old stre
github.tar, level 7 with dict, old streaming advanced cdict, 37322 github.tar, level 7 with dict, old streaming advanced cdict, 37322
github.tar, level 9 with dict, old streaming advanced cdict, 36233 github.tar, level 9 with dict, old streaming advanced cdict, 36233
github.tar, level 13 with dict, old streaming advanced cdict, 35807 github.tar, level 13 with dict, old streaming advanced cdict, 35807
github.tar, level 16 with dict, old streaming advanced cdict, 38736 github.tar, level 16 with dict, old streaming advanced cdict, 38578
github.tar, level 19 with dict, old streaming advanced cdict, 32876 github.tar, level 19 with dict, old streaming advanced cdict, 32702
github.tar, no source size with dict, old streaming advanced cdict, 38015 github.tar, no source size with dict, old streaming advanced cdict, 38015
1 Data Config Method Total compressed size
11 silesia.tar level 7 compress simple 4576829
12 silesia.tar level 9 compress simple 4552584
13 silesia.tar level 13 compress simple 4502956
14 silesia.tar level 16 compress simple 4356834 4360527
15 silesia.tar level 19 compress simple 4264388 4267266
16 silesia.tar uncompressed literals compress simple 4861424
17 silesia.tar uncompressed literals optimal compress simple 4264388 4267266
18 silesia.tar huffman literals compress simple 6182241
19 github.tar level -5 compress simple 66914
20 github.tar level -3 compress simple 52127
28 github.tar level 7 compress simple 38073
29 github.tar level 9 compress simple 36767
30 github.tar level 13 compress simple 35501
31 github.tar level 16 compress simple 40255 40471
32 github.tar level 19 compress simple 32837 32134
33 github.tar uncompressed literals compress simple 38441
34 github.tar uncompressed literals optimal compress simple 32837 32134
35 github.tar huffman literals compress simple 42560
36 silesia level -5 compress cctx 7354675
37 silesia level -3 compress cctx 6902374
45 silesia level 7 compress cctx 4567204
46 silesia level 9 compress cctx 4543310
47 silesia level 13 compress cctx 4493990
48 silesia level 16 compress cctx 4360251 4359864
49 silesia level 19 compress cctx 4283237 4296880
50 silesia long distance mode compress cctx 4849553
51 silesia multithreaded compress cctx 4849553
52 silesia multithreaded long distance mode compress cctx 4849553
55 silesia small chain log compress cctx 4912197
56 silesia explicit params compress cctx 4794481
57 silesia uncompressed literals compress cctx 4849553
58 silesia uncompressed literals optimal compress cctx 4283237 4296880
59 silesia huffman literals compress cctx 6177565
60 silesia multithreaded with advanced params compress cctx 4849553
61 github level -5 compress cctx 232315
109 silesia level 7 zstdcli 4567252
110 silesia level 9 zstdcli 4543358
111 silesia level 13 zstdcli 4494038
112 silesia level 16 zstdcli 4360299 4359912
113 silesia level 19 zstdcli 4283285 4296928
114 silesia long distance mode zstdcli 4840807
115 silesia multithreaded zstdcli 4849601
116 silesia multithreaded long distance mode zstdcli 4840807
119 silesia small chain log zstdcli 4912245
120 silesia explicit params zstdcli 4795856
121 silesia uncompressed literals zstdcli 5128030
122 silesia uncompressed literals optimal zstdcli 4317944 4319566
123 silesia huffman literals zstdcli 5326394
124 silesia multithreaded with advanced params zstdcli 5128030
125 silesia.tar level -5 zstdcli 7363866
134 silesia.tar level 7 zstdcli 4578884
135 silesia.tar level 9 zstdcli 4553499
136 silesia.tar level 13 zstdcli 4502960
137 silesia.tar level 16 zstdcli 4356838 4360531
138 silesia.tar level 19 zstdcli 4264392 4267270
139 silesia.tar no source size zstdcli 4861508
140 silesia.tar long distance mode zstdcli 4853225
141 silesia.tar multithreaded zstdcli 4861512
145 silesia.tar small chain log zstdcli 4917022
146 silesia.tar explicit params zstdcli 4821274
147 silesia.tar uncompressed literals zstdcli 5129559
148 silesia.tar uncompressed literals optimal zstdcli 4307404 4310145
149 silesia.tar huffman literals zstdcli 5344915
150 silesia.tar multithreaded with advanced params zstdcli 5129559
151 github level -5 zstdcli 234315
210 github.tar level 9 zstdcli 36771
211 github.tar level 9 with dict zstdcli 36623
212 github.tar level 13 zstdcli 35505
213 github.tar level 13 with dict zstdcli 38730 37134
214 github.tar level 16 zstdcli 40259 40475
215 github.tar level 16 with dict zstdcli 33643 33382
216 github.tar level 19 zstdcli 32841 32138
217 github.tar level 19 with dict zstdcli 32899 32713
218 github.tar no source size zstdcli 38442
219 github.tar no source size with dict zstdcli 38004
220 github.tar long distance mode zstdcli 39730
225 github.tar small chain log zstdcli 41673
226 github.tar explicit params zstdcli 41227
227 github.tar uncompressed literals zstdcli 41126
228 github.tar uncompressed literals optimal zstdcli 35392 35401
229 github.tar huffman literals zstdcli 38857
230 github.tar multithreaded with advanced params zstdcli 41126
231 silesia level -5 advanced one pass 7354675
248 silesia level 12 row 1 advanced one pass 4503116
249 silesia level 12 row 2 advanced one pass 4505153
250 silesia level 13 advanced one pass 4493990
251 silesia level 16 advanced one pass 4360251 4359864
252 silesia level 19 advanced one pass 4283237 4296880
253 silesia no source size advanced one pass 4849553
254 silesia long distance mode advanced one pass 4840737
255 silesia multithreaded advanced one pass 4849553
259 silesia small chain log advanced one pass 4912197
260 silesia explicit params advanced one pass 4795856
261 silesia uncompressed literals advanced one pass 5127982
262 silesia uncompressed literals optimal advanced one pass 4317896 4319518
263 silesia huffman literals advanced one pass 5326346
264 silesia multithreaded with advanced params advanced one pass 5127982
265 silesia.tar level -5 advanced one pass 7359401
282 silesia.tar level 12 row 1 advanced one pass 4513604
283 silesia.tar level 12 row 2 advanced one pass 4514568
284 silesia.tar level 13 advanced one pass 4502956
285 silesia.tar level 16 advanced one pass 4356834 4360527
286 silesia.tar level 19 advanced one pass 4264388 4267266
287 silesia.tar no source size advanced one pass 4861424
288 silesia.tar long distance mode advanced one pass 4847752
289 silesia.tar multithreaded advanced one pass 4861508
293 silesia.tar small chain log advanced one pass 4917041
294 silesia.tar explicit params advanced one pass 4807381
295 silesia.tar uncompressed literals advanced one pass 5129458
296 silesia.tar uncompressed literals optimal advanced one pass 4307400 4310141
297 silesia.tar huffman literals advanced one pass 5344545
298 silesia.tar multithreaded with advanced params advanced one pass 5129555
299 github level -5 advanced one pass 232315
516 github.tar level 12 row 2 with dict copy advanced one pass 36609
517 github.tar level 12 row 2 with dict load advanced one pass 36460
518 github.tar level 13 advanced one pass 35501
519 github.tar level 13 with dict advanced one pass 38726 37130
520 github.tar level 13 with dict dms advanced one pass 38903 37267
521 github.tar level 13 with dict dds advanced one pass 38903 37267
522 github.tar level 13 with dict copy advanced one pass 38726 37130
523 github.tar level 13 with dict load advanced one pass 36010
524 github.tar level 16 advanced one pass 40255 40471
525 github.tar level 16 with dict advanced one pass 33639 33378
526 github.tar level 16 with dict dms advanced one pass 33544 33213
527 github.tar level 16 with dict dds advanced one pass 33544 33213
528 github.tar level 16 with dict copy advanced one pass 33639 33378
529 github.tar level 16 with dict load advanced one pass 39353 39081
530 github.tar level 19 advanced one pass 32837 32134
531 github.tar level 19 with dict advanced one pass 32895 32709
532 github.tar level 19 with dict dms advanced one pass 32672 32553
533 github.tar level 19 with dict dds advanced one pass 32672 32553
534 github.tar level 19 with dict copy advanced one pass 32895 32709
535 github.tar level 19 with dict load advanced one pass 32676 32474
536 github.tar no source size advanced one pass 38441
537 github.tar no source size with dict advanced one pass 37995
538 github.tar long distance mode advanced one pass 39757
543 github.tar small chain log advanced one pass 41669
544 github.tar explicit params advanced one pass 41227
545 github.tar uncompressed literals advanced one pass 41122
546 github.tar uncompressed literals optimal advanced one pass 35388 35397
547 github.tar huffman literals advanced one pass 38853
548 github.tar multithreaded with advanced params advanced one pass 41122
549 silesia level -5 advanced one pass small out 7354675
566 silesia level 12 row 1 advanced one pass small out 4503116
567 silesia level 12 row 2 advanced one pass small out 4505153
568 silesia level 13 advanced one pass small out 4493990
569 silesia level 16 advanced one pass small out 4360251 4359864
570 silesia level 19 advanced one pass small out 4283237 4296880
571 silesia no source size advanced one pass small out 4849553
572 silesia long distance mode advanced one pass small out 4840737
573 silesia multithreaded advanced one pass small out 4849553
577 silesia small chain log advanced one pass small out 4912197
578 silesia explicit params advanced one pass small out 4795856
579 silesia uncompressed literals advanced one pass small out 5127982
580 silesia uncompressed literals optimal advanced one pass small out 4317896 4319518
581 silesia huffman literals advanced one pass small out 5326346
582 silesia multithreaded with advanced params advanced one pass small out 5127982
583 silesia.tar level -5 advanced one pass small out 7359401
600 silesia.tar level 12 row 1 advanced one pass small out 4513604
601 silesia.tar level 12 row 2 advanced one pass small out 4514568
602 silesia.tar level 13 advanced one pass small out 4502956
603 silesia.tar level 16 advanced one pass small out 4356834 4360527
604 silesia.tar level 19 advanced one pass small out 4264388 4267266
605 silesia.tar no source size advanced one pass small out 4861424
606 silesia.tar long distance mode advanced one pass small out 4847752
607 silesia.tar multithreaded advanced one pass small out 4861508
611 silesia.tar small chain log advanced one pass small out 4917041
612 silesia.tar explicit params advanced one pass small out 4807381
613 silesia.tar uncompressed literals advanced one pass small out 5129458
614 silesia.tar uncompressed literals optimal advanced one pass small out 4307400 4310141
615 silesia.tar huffman literals advanced one pass small out 5344545
616 silesia.tar multithreaded with advanced params advanced one pass small out 5129555
617 github level -5 advanced one pass small out 232315
834 github.tar level 12 row 2 with dict copy advanced one pass small out 36609
835 github.tar level 12 row 2 with dict load advanced one pass small out 36460
836 github.tar level 13 advanced one pass small out 35501
837 github.tar level 13 with dict advanced one pass small out 38726 37130
838 github.tar level 13 with dict dms advanced one pass small out 38903 37267
839 github.tar level 13 with dict dds advanced one pass small out 38903 37267
840 github.tar level 13 with dict copy advanced one pass small out 38726 37130
841 github.tar level 13 with dict load advanced one pass small out 36010
842 github.tar level 16 advanced one pass small out 40255 40471
843 github.tar level 16 with dict advanced one pass small out 33639 33378
844 github.tar level 16 with dict dms advanced one pass small out 33544 33213
845 github.tar level 16 with dict dds advanced one pass small out 33544 33213
846 github.tar level 16 with dict copy advanced one pass small out 33639 33378
847 github.tar level 16 with dict load advanced one pass small out 39353 39081
848 github.tar level 19 advanced one pass small out 32837 32134
849 github.tar level 19 with dict advanced one pass small out 32895 32709
850 github.tar level 19 with dict dms advanced one pass small out 32672 32553
851 github.tar level 19 with dict dds advanced one pass small out 32672 32553
852 github.tar level 19 with dict copy advanced one pass small out 32895 32709
853 github.tar level 19 with dict load advanced one pass small out 32676 32474
854 github.tar no source size advanced one pass small out 38441
855 github.tar no source size with dict advanced one pass small out 37995
856 github.tar long distance mode advanced one pass small out 39757
861 github.tar small chain log advanced one pass small out 41669
862 github.tar explicit params advanced one pass small out 41227
863 github.tar uncompressed literals advanced one pass small out 41122
864 github.tar uncompressed literals optimal advanced one pass small out 35388 35397
865 github.tar huffman literals advanced one pass small out 38853
866 github.tar multithreaded with advanced params advanced one pass small out 41122
867 silesia level -5 advanced streaming 7292053
884 silesia level 12 row 1 advanced streaming 4503116
885 silesia level 12 row 2 advanced streaming 4505153
886 silesia level 13 advanced streaming 4493990
887 silesia level 16 advanced streaming 4360251 4359864
888 silesia level 19 advanced streaming 4283237 4296880
889 silesia no source size advanced streaming 4849517
890 silesia long distance mode advanced streaming 4840737
891 silesia multithreaded advanced streaming 4849553
895 silesia small chain log advanced streaming 4912197
896 silesia explicit params advanced streaming 4795883
897 silesia uncompressed literals advanced streaming 5127982
898 silesia uncompressed literals optimal advanced streaming 4317896 4319518
899 silesia huffman literals advanced streaming 5332234
900 silesia multithreaded with advanced params advanced streaming 5127982
901 silesia.tar level -5 advanced streaming 7260007
918 silesia.tar level 12 row 1 advanced streaming 4513604
919 silesia.tar level 12 row 2 advanced streaming 4514569
920 silesia.tar level 13 advanced streaming 4502956
921 silesia.tar level 16 advanced streaming 4356834 4360527
922 silesia.tar level 19 advanced streaming 4264388 4267266
923 silesia.tar no source size advanced streaming 4861422
924 silesia.tar long distance mode advanced streaming 4847752
925 silesia.tar multithreaded advanced streaming 4861508
929 silesia.tar small chain log advanced streaming 4917021
930 silesia.tar explicit params advanced streaming 4807401
931 silesia.tar uncompressed literals advanced streaming 5129461
932 silesia.tar uncompressed literals optimal advanced streaming 4307400 4310141
933 silesia.tar huffman literals advanced streaming 5350519
934 silesia.tar multithreaded with advanced params advanced streaming 5129555
935 github level -5 advanced streaming 232315
1152 github.tar level 12 row 2 with dict copy advanced streaming 36609
1153 github.tar level 12 row 2 with dict load advanced streaming 36460
1154 github.tar level 13 advanced streaming 35501
1155 github.tar level 13 with dict advanced streaming 38726 37130
1156 github.tar level 13 with dict dms advanced streaming 38903 37267
1157 github.tar level 13 with dict dds advanced streaming 38903 37267
1158 github.tar level 13 with dict copy advanced streaming 38726 37130
1159 github.tar level 13 with dict load advanced streaming 36010
1160 github.tar level 16 advanced streaming 40255 40471
1161 github.tar level 16 with dict advanced streaming 33639 33378
1162 github.tar level 16 with dict dms advanced streaming 33544 33213
1163 github.tar level 16 with dict dds advanced streaming 33544 33213
1164 github.tar level 16 with dict copy advanced streaming 33639 33378
1165 github.tar level 16 with dict load advanced streaming 39353 39081
1166 github.tar level 19 advanced streaming 32837 32134
1167 github.tar level 19 with dict advanced streaming 32895 32709
1168 github.tar level 19 with dict dms advanced streaming 32672 32553
1169 github.tar level 19 with dict dds advanced streaming 32672 32553
1170 github.tar level 19 with dict copy advanced streaming 32895 32709
1171 github.tar level 19 with dict load advanced streaming 32676 32474
1172 github.tar no source size advanced streaming 38438
1173 github.tar no source size with dict advanced streaming 38000
1174 github.tar long distance mode advanced streaming 39757
1179 github.tar small chain log advanced streaming 41669
1180 github.tar explicit params advanced streaming 41227
1181 github.tar uncompressed literals advanced streaming 41122
1182 github.tar uncompressed literals optimal advanced streaming 35388 35397
1183 github.tar huffman literals advanced streaming 38874
1184 github.tar multithreaded with advanced params advanced streaming 41122
1185 silesia level -5 old streaming 7292053
1194 silesia level 7 old streaming 4567204
1195 silesia level 9 old streaming 4543310
1196 silesia level 13 old streaming 4493990
1197 silesia level 16 old streaming 4360251 4359864
1198 silesia level 19 old streaming 4283237 4296880
1199 silesia no source size old streaming 4849517
1200 silesia uncompressed literals old streaming 4849553
1201 silesia uncompressed literals optimal old streaming 4283237 4296880
1202 silesia huffman literals old streaming 6183923
1203 silesia.tar level -5 old streaming 7260007
1204 silesia.tar level -3 old streaming 6845151
1212 silesia.tar level 7 old streaming 4576831
1213 silesia.tar level 9 old streaming 4552590
1214 silesia.tar level 13 old streaming 4502956
1215 silesia.tar level 16 old streaming 4356834 4360527
1216 silesia.tar level 19 old streaming 4264388 4267266
1217 silesia.tar no source size old streaming 4861422
1218 silesia.tar uncompressed literals old streaming 4861426
1219 silesia.tar uncompressed literals optimal old streaming 4264388 4267266
1220 silesia.tar huffman literals old streaming 6187938
1221 github level -5 old streaming 232315
1222 github level -5 with dict old streaming 46718
1274 github.tar level 9 old streaming 36767
1275 github.tar level 9 with dict old streaming 36457
1276 github.tar level 13 old streaming 35501
1277 github.tar level 13 with dict old streaming 38726 37130
1278 github.tar level 16 old streaming 40255 40471
1279 github.tar level 16 with dict old streaming 33639 33378
1280 github.tar level 19 old streaming 32837 32134
1281 github.tar level 19 with dict old streaming 32895 32709
1282 github.tar no source size old streaming 38438
1283 github.tar no source size with dict old streaming 38000
1284 github.tar uncompressed literals old streaming 38441
1285 github.tar uncompressed literals optimal old streaming 32837 32134
1286 github.tar huffman literals old streaming 42536
1287 silesia level -5 old streaming advanced 7292053
1288 silesia level -3 old streaming advanced 6867875
1296 silesia level 7 old streaming advanced 4567204
1297 silesia level 9 old streaming advanced 4543310
1298 silesia level 13 old streaming advanced 4493990
1299 silesia level 16 old streaming advanced 4360251 4359864
1300 silesia level 19 old streaming advanced 4283237 4296880
1301 silesia no source size old streaming advanced 4849517
1302 silesia long distance mode old streaming advanced 4849553
1303 silesia multithreaded old streaming advanced 4849553
1307 silesia small chain log old streaming advanced 4912197
1308 silesia explicit params old streaming advanced 4795883
1309 silesia uncompressed literals old streaming advanced 4849553
1310 silesia uncompressed literals optimal old streaming advanced 4283237 4296880
1311 silesia huffman literals old streaming advanced 6183923
1312 silesia multithreaded with advanced params old streaming advanced 4849553
1313 silesia.tar level -5 old streaming advanced 7260007
1322 silesia.tar level 7 old streaming advanced 4576831
1323 silesia.tar level 9 old streaming advanced 4552590
1324 silesia.tar level 13 old streaming advanced 4502956
1325 silesia.tar level 16 old streaming advanced 4356834 4360527
1326 silesia.tar level 19 old streaming advanced 4264388 4267266
1327 silesia.tar no source size old streaming advanced 4861422
1328 silesia.tar long distance mode old streaming advanced 4861426
1329 silesia.tar multithreaded old streaming advanced 4861426
1333 silesia.tar small chain log old streaming advanced 4917021
1334 silesia.tar explicit params old streaming advanced 4807401
1335 silesia.tar uncompressed literals old streaming advanced 4861426
1336 silesia.tar uncompressed literals optimal old streaming advanced 4264388 4267266
1337 silesia.tar huffman literals old streaming advanced 6187938
1338 silesia.tar multithreaded with advanced params old streaming advanced 4861426
1339 github level -5 old streaming advanced 241214
1401 github.tar level 9 with dict old streaming advanced 36233
1402 github.tar level 13 old streaming advanced 35501
1403 github.tar level 13 with dict old streaming advanced 35807
1404 github.tar level 16 old streaming advanced 40255 40471
1405 github.tar level 16 with dict old streaming advanced 38736 38578
1406 github.tar level 19 old streaming advanced 32837 32134
1407 github.tar level 19 with dict old streaming advanced 32876 32702
1408 github.tar no source size old streaming advanced 38438
1409 github.tar no source size with dict old streaming advanced 38015
1410 github.tar long distance mode old streaming advanced 38441
1415 github.tar small chain log old streaming advanced 41669
1416 github.tar explicit params old streaming advanced 41227
1417 github.tar uncompressed literals old streaming advanced 38441
1418 github.tar uncompressed literals optimal old streaming advanced 32837 32134
1419 github.tar huffman literals old streaming advanced 42536
1420 github.tar multithreaded with advanced params old streaming advanced 38441
1421 github level -5 with dict old streaming cdict 46718
1445 github.tar level 7 with dict old streaming cdict 37371
1446 github.tar level 9 with dict old streaming cdict 36352
1447 github.tar level 13 with dict old streaming cdict 36010
1448 github.tar level 16 with dict old streaming cdict 39353 39081
1449 github.tar level 19 with dict old streaming cdict 32676 32474
1450 github.tar no source size with dict old streaming cdict 38000
1451 github level -5 with dict old streaming advanced cdict 49562
1452 github level -3 with dict old streaming advanced cdict 44956
1475 github.tar level 7 with dict old streaming advanced cdict 37322
1476 github.tar level 9 with dict old streaming advanced cdict 36233
1477 github.tar level 13 with dict old streaming advanced cdict 35807
1478 github.tar level 16 with dict old streaming advanced cdict 38736 38578
1479 github.tar level 19 with dict old streaming advanced cdict 32876 32702
1480 github.tar no source size with dict old streaming advanced cdict 38015