ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changed price estimation for optimal parser

Browse Source
This commit is contained in:
inikep
2016-04-04 14:34:02 +02:00
parent 19bd48f516
commit 227dd4bc74
1 changed files with 54 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_opt.h
+54 -30
View File
@@ -111,9 +111,18 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1); price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1);
/* literal Length */ /* literal Length */
price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
if (litLength >= MaxLL) litLength = MaxLL; 8, 9, 10, 11, 12, 13, 14, 15,
price += seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1); 16, 16, 17, 17, 18, 18, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1);
}
return price; return price;
} }
@@ -122,25 +131,24 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
{ {
/* offset */ /* offset */
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
U32 price = (offCode-1) + (!offCode) + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
/* match Length */ /* match Length */
price += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
if (matchLength >= MaxML) matchLength = MaxML; 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
#if ZSTD_OPT_DEBUG == 333 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
switch (seqStorePtr->) { 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
default: 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
case 0: 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); const BYTE ML_deltaCode = 36;
case 1: const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
return 1 + price; price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1);
} }
#else
return price + seqStorePtr->factor; return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
#endif
} }
@@ -154,23 +162,39 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
seqStorePtr->litFreq[literals[u]]++; seqStorePtr->litFreq[literals[u]]++;
/* literal Length */ /* literal Length */
seqStorePtr->litLengthSum++; { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
if (litLength >= MaxLL) 8, 9, 10, 11, 12, 13, 14, 15,
seqStorePtr->litLengthFreq[MaxLL]++; 16, 16, 17, 17, 18, 18, 19, 19,
else 20, 20, 20, 20, 21, 21, 21, 21,
seqStorePtr->litLengthFreq[litLength]++; 22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
seqStorePtr->litLengthFreq[llCode]++;
seqStorePtr->litLengthSum++;
}
/* match offset */ /* match offset */
seqStorePtr->offCodeSum++; seqStorePtr->offCodeSum++;
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; BYTE offCode = ZSTD_highbit(offset+1);
seqStorePtr->offCodeFreq[offCode]++; seqStorePtr->offCodeFreq[offCode]++;
/* match Length */ /* match Length */
seqStorePtr->matchLengthSum++; { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
if (matchLength >= MaxML) 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
seqStorePtr->matchLengthFreq[MaxML]++; 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
else 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
seqStorePtr->matchLengthFreq[matchLength]++; 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
seqStorePtr->matchLengthFreq[mlCode]++;
seqStorePtr->matchLengthSum++;
}
ZSTD_setLog2Prices(seqStorePtr); ZSTD_setLog2Prices(seqStorePtr);
} }