ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pass entropy tables to optimal parser

Browse Source 
for proper estimation of symbol's weights
when using dictionary compression.

Note : using only huffman costs is not good enough,
presumably because sequence symbol costs are incorrect.
This commit is contained in:
Yann Collet
2018-05-08 15:37:06 -07:00
parent a155061328
commit 338f738c24
5 changed files with 39 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_opt.c
+18 -6
View File
@@ -32,11 +32,15 @@ static void ZSTD_setLog2Prices(optState_t* optPtr)
static void ZSTD_rescaleFreqs(optState_t* const optPtr,
const BYTE* const src, size_t const srcSize)
{
optPtr->predefPrices = 0;
optPtr->priceType = zop_none;
if (optPtr->litLengthSum == 0) { /* first init */
unsigned u;
if (srcSize <= 1024) optPtr->predefPrices = 1;
if (srcSize <= 1024) optPtr->priceType = zop_predef;
assert(optPtr->symbolCosts != NULL);
if (0 && optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
optPtr->priceType = zop_static;
}
assert(optPtr->litFreq!=NULL);
for (u=0; u<=MaxLit; u++)
@@ -94,7 +98,15 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
const optState_t* const optPtr)
{
if (optPtr->predefPrices) return (litLength*6); /* 6 bit per literal - no statistic used */
if (optPtr->priceType == zop_static) {
U32 u, cost;
assert(optPtr->symbolCosts != NULL);
assert(optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid);
for (u=0, cost=0; u < litLength; u++)
cost += HUF_getNbBits(optPtr->symbolCosts->hufCTable, literals[u]);
return cost;
}
if (optPtr->priceType == zop_predef) return (litLength*6); /* 6 bit per literal - no statistic used */
if (litLength == 0) return 0;
/* literals */
@@ -110,7 +122,7 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
* cost of literalLength symbol */
static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr)
{
if (optPtr->predefPrices) return ZSTD_highbit32((U32)litLength+1);
if (optPtr->priceType == zop_predef) return ZSTD_highbit32((U32)litLength+1);
/* literal Length */
{ U32 const llCode = ZSTD_LLcode(litLength);
@@ -135,7 +147,7 @@ static U32 ZSTD_fullLiteralsCost(const BYTE* const literals, U32 const litLength
* to provide a cost which is directly comparable to a match ending at same position */
static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr)
{
if (optPtr->predefPrices) return ZSTD_highbit32(litLength+1);
if (optPtr->priceType == zop_predef) return ZSTD_highbit32(litLength+1);
/* literal Length */
{ U32 const llCode = ZSTD_LLcode(litLength);
@@ -176,7 +188,7 @@ ZSTD_getMatchPrice(U32 const offset, U32 const matchLength,
U32 const mlBase = matchLength - MINMATCH;
assert(matchLength >= MINMATCH);
if (optPtr->predefPrices) /* fixed scheme, do not use statistics */
if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */
return ZSTD_highbit32(mlBase+1) + 16 + offCode;
price = offCode + optPtr->log2offCodeSum - ZSTD_highbit32(optPtr->offCodeFreq[offCode]+1);