ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

modification: differentiate literal update at pos+1

Browse Source 
helps when litlen==1 is cheaper than litlen==0

works great on pathological arr[u32] examples
but doesn't generalize well on other files.

silesia/x-ray is amoung the most negatively affected ones.
This commit is contained in:
Yann Collet
2024-01-31 11:20:43 -08:00
parent 4683667785
commit 0166b2ba80
1 changed files with 27 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_opt.c
+27 -25
View File
@@ -1200,8 +1200,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
if ( 0 && (optLevel == 2) /* additional check only for high modes */ if ((optLevel == 2) /* additional check only for high modes */
&& (opt[cur].mlen > 0) /* interrupt a match */ && (opt[cur].litlen == 0) /* interrupt a match */
&& (LL_INCPRICE(1) < 0) ) /* ll1 is cheaper than ll0 */ && (LL_INCPRICE(1) < 0) ) /* ll1 is cheaper than ll0 */
{ {
/* check next position, in case it would be cheaper */ /* check next position, in case it would be cheaper */
@@ -1209,33 +1209,35 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
int withMoreLiterals = price + LL_INCPRICE(litlen+1); int withMoreLiterals = price + LL_INCPRICE(litlen+1);
DEBUGLOG(7, "But at next rPos %u : match+1lit %.2f vs %ulits %.2f", DEBUGLOG(7, "But at next rPos %u : match+1lit %.2f vs %ulits %.2f",
cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals)); cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals));
if (with1literal < withMoreLiterals) { if ( (with1literal < withMoreLiterals)
DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) !!!", ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals)); && (with1literal < opt[cur+1].price) ) {
/* do not take this literal */ /* update offset history - before it disappears */
} else { U32 const prev = cur - opt[cur].mlen;
opt[cur].mlen = opt[cur-1].mlen; repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
opt[cur].off = opt[cur-1].off; assert(cur >= opt[cur].mlen);
opt[cur].litlen = litlen; DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !",
opt[cur].price = price; ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals),
newReps.rep[0], newReps.rep[1], newReps.rep[2] );
opt[cur+1] = opt[cur]; /* mlen & offbase */
ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(repcodes_t));
opt[cur+1].litlen = 1;
opt[cur+1].price = with1literal;
if (last_pos < cur+1) last_pos = cur+1;
} }
} else {
/* normal case: take the literal, it's expected to be cheaper at position @cur */
opt[cur].mlen = opt[cur-1].mlen;
opt[cur].off = opt[cur-1].off;
opt[cur].litlen = litlen;
opt[cur].price = price;
} }
opt[cur].mlen = opt[cur-1].mlen;
opt[cur].off = opt[cur-1].off;
opt[cur].litlen = litlen;
opt[cur].price = price;
ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
} else { } else {
DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]);
} }
} }
/* Set the repcodes of the current position. We must do it here /* Offset history is not updated during match comparison.
* because we rely on the repcodes of the 2nd to last sequence being * Do it here, now that the match is selected and confirmed.
* correct to set the next chunks repcodes during the backward
* traversal.
*/ */
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
assert(cur >= opt[cur].mlen); assert(cur >= opt[cur].mlen);
@@ -1244,8 +1246,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 const prev = cur - opt[cur].mlen; U32 const prev = cur - opt[cur].mlen;
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0); repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
} else {
ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
} }
/* last match must start at a minimum distance of 8 from oend */ /* last match must start at a minimum distance of 8 from oend */
@@ -1514,6 +1514,7 @@ size_t ZSTD_compressBlock_btultra2(
* The compression ratio gain is generally small (~0.5% on first block), * The compression ratio gain is generally small (~0.5% on first block),
* the cost is 2x cpu time on first block. */ * the cost is 2x cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX); assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
//g_debuglevel = -g_debuglevel;
if ( (ms->opt.litLengthSum==0) /* first block */ if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
@@ -1523,6 +1524,7 @@ size_t ZSTD_compressBlock_btultra2(
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
} }
//g_debuglevel = -g_debuglevel;
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
} }
#endif #endif