diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 3d16d087e..f06f55639 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -146,7 +146,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - { U32 const searchLengthMin = (cParams.strategy == ZSTD_btopt) ? ZSTD_SEARCHLENGTH_MIN : ZSTD_SEARCHLENGTH_MIN+1; + { U32 const searchLengthMin = (cParams.strategy == ZSTD_fast || cParams.strategy == ZSTD_greedy) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); @@ -1058,25 +1058,26 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, hashTable[h] = current; /* update hash table */ if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) { /* note : by construction, offset_1 <= current */ - mlCode = ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-offset_1, iend); + mlCode = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; ip++; - offset = 0; - } else { + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH); + } else { if ( (matchIndex <= lowIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; } - mlCode = ZSTD_count(ip+MINMATCH, match+MINMATCH, iend); + mlCode = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32; offset = ip-match; while ((ip>anchor) && (match>lowest) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH); } /* match found */ - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset, mlCode); - ip += mlCode + MINMATCH; + ip += mlCode; anchor = ip; if (ip <= ilimit) { @@ -1087,11 +1088,11 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, while ( (ip <= ilimit) && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { /* store sequence */ - size_t const rlCode = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend); + size_t const rlCode = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32; { size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rlCode); - ip += rlCode+MINMATCH; + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rlCode-MINMATCH); + ip += rlCode; anchor = ip; continue; /* faster when present ... (?) */ } } } @@ -1169,9 +1170,9 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, if ( ((repIndex >= dictLimit) || (repIndex <= dictLimit-4)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; - mlCode = ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repMatchEnd, lowPrefixPtr); + mlCode = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; ip++; - offset = 0; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH); } else { if ( (matchIndex < lowLimit) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -1180,16 +1181,16 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, } { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; - mlCode = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iend, matchEnd, lowPrefixPtr); + mlCode = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; while ((ip>anchor) && (match>lowMatchPtr) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; } /* catch up */ offset = current - matchIndex; offset_2 = offset_1; offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH); } } /* found a match : store it */ - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset, mlCode); - ip += mlCode + MINMATCH; + ip += mlCode; anchor = ip; if (ip <= ilimit) { @@ -1204,11 +1205,11 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, if ( ((repIndex2 <= dictLimit-4) || (repIndex2 >= dictLimit)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t repLength2 = ZSTD_count_2segments(ip+MINMATCH, repMatch2+MINMATCH, iend, repEnd2, lowPrefixPtr); + size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; - ip += repLength2+MINMATCH; + ip += repLength2; anchor = ip; continue; } @@ -1242,6 +1243,167 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, } + + +/* *********************** +* Hash Chain +*************************/ + +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) +{ + U32* const hashTable = zc->hashTable; + const U32 hashLog = zc->params.cParams.hashLog; + U32* const chainTable = zc->chainTable; + const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) { + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + zc->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) +{ + U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; + const BYTE* const base = zc->base; + U32 idx = zc->nextToUpdate3; + const U32 target = zc->nextToUpdate3 = (U32)(ip - base); + const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + return hashTable3[hash3]; +} + + +FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +size_t ZSTD_HcFindBestMatch_generic ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls, const U32 extDict) +{ + U32* const chainTable = zc->chainTable; + const U32 chainSize = (1 << zc->params.cParams.chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 lowLimit = zc->lowLimit; + const U32 current = (U32)(ip-base); + const U32 minChain = current > chainSize ? current - chainSize : 0; + int nbAttempts=maxNbAttempts; + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t ml=minMatch-1; + +#if 0 + if (minMatch == 3) { /* HC3 match finder */ + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (matchIndex3>lowLimit && current - matchIndex3<(1<<18)) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[ml] == ip[ml]) currentMl = ZSTD_count(ip, match, iLimit); /* potentially better */ + } else { + match = dictBase + matchIndex3; + if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex3; + if (ip+currentMl == iLimit) return (ml>=MINMATCH) ? ml : 0; /* best possible, and avoid read overflow*/ + } } + } +#endif + + /* HC4 match finder */ + U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); + + for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex >= dictLimit) { + match = base + matchIndex; + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex; + if (MEM_readMINMATCH(match, minMatch) == MEM_readMINMATCH(ip, minMatch)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+minMatch, match+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + + /* save best solution */ + if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + return (ml>=minMatch) ? ml : 0; +} + + +FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 3, 0); + default : + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); + } +} + + +FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 3, 1); + default : + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); + } +} + + + /*-************************************* * Binary Tree search ***************************************/ @@ -1371,9 +1533,31 @@ static size_t ZSTD_insertBtAndFindBestMatch ( const U32 windowLow = zc->lowLimit; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; - size_t bestLength = 0; U32 matchEndIdx = current+8; U32 dummy32; /* to be nullified at the end */ + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t bestLength = minMatch-1; + + if (minMatch == 3) { /* HC3 match finder */ + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iend); + } else { + match = dictBase + matchIndex3; + if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iend, dictEnd, prefixStart) + MINMATCH; + } + + /* save best solution */ + if (currentMl > bestLength) { + bestLength = currentMl, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex3; + if (ip+currentMl == iend) goto update; /* best possible, and avoid read overflow*/ + } + } } hashTable[h] = current; /* Update Hash Table */ @@ -1397,7 +1581,7 @@ static size_t ZSTD_insertBtAndFindBestMatch ( if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) - bestLength = matchLength, *offsetPtr = current - matchIndex; + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ } @@ -1420,8 +1604,9 @@ static size_t ZSTD_insertBtAndFindBestMatch ( *smallerPtr = *largerPtr = 0; +update: zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; - return bestLength; + return (bestLength>=minMatch) ? bestLength : 0; } @@ -1456,6 +1641,7 @@ static size_t ZSTD_BtFindBestMatch_selectMLS ( { switch(matchLengthSearch) { + case 3 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 3); default : case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); @@ -1473,8 +1659,6 @@ static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const B while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); } -#include "zstd_opt.h" - /** Tree updater, providing best match */ static size_t ZSTD_BtFindBestMatch_extDict ( ZSTD_CCtx* zc, @@ -1496,6 +1680,7 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( { switch(matchLengthSearch) { + case 3 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 3); default : case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); @@ -1504,117 +1689,194 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( } -/* *********************** -* Hash Chain -*************************/ -#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] - -/* Update chains up to ip (excluded) - Assumption : always within prefix (ie. not within extDict) */ +/* ******************************* +* Greedy parser +*********************************/ FORCE_INLINE -U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) +void ZSTD_compressBlock_greedy_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) { - U32* const hashTable = zc->hashTable; - const U32 hashLog = zc->params.cParams.hashLog; - U32* const chainTable = zc->chainTable; - const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; - const BYTE* const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base + ctx->dictLimit; - while(idx < target) { - size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); - NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; - hashTable[h] = idx; - idx++; - } + const U32 maxSearches = 1 << ctx->params.cParams.searchLog; + const U32 mls = ctx->params.cParams.searchLength; - zc->nextToUpdate = target; - return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; -} + /* init */ + U32 rep[ZSTD_REP_INIT]; + for (U32 i=0; ichainTable; - const U32 chainSize = (1 << zc->params.cParams.chainLog); - const U32 chainMask = chainSize-1; - const BYTE* const base = zc->base; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = zc->lowLimit; - const U32 current = (U32)(ip-base); - const U32 minChain = current > chainSize ? current - chainSize : 0; - U32 matchIndex; - const BYTE* match; - int nbAttempts=maxNbAttempts; - size_t ml=MINMATCH-1; + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; - /* HC4 match finder */ - matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); - - for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) { - size_t currentMl=0; - if ((!extDict) || matchIndex >= dictLimit) { - match = base + matchIndex; - if (match[ml] == ip[ml]) /* potentially better */ - currentMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex; - if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; + /* check repCode */ + if (MEM_read32(start) == MEM_read32(start - rep[0])) { + /* repcode : we take it */ + matchLength = ZSTD_count(start+EQUAL_READ32, start+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; + goto _storeSequence; } - /* save best solution */ - if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = ZSTD_HcFindBestMatch_selectMLS(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } - if (matchIndex <= minChain) break; - matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + if (matchLength < EQUAL_READ32) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* catch up */ + while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ + { start--; matchLength++; } + rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE); + +_storeSequence: + /* store sequence */ + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && (MEM_read32(ip) == MEM_read32(ip - rep[1])) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[1], iend) + EQUAL_READ32; + offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } - - return ml; } -FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( - ZSTD_CCtx* zc, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 matchLengthSearch) +FORCE_INLINE +void ZSTD_compressBlock_greedy_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) { - switch(matchLengthSearch) - { - default : - case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); - case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); - case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ctx->lowLimit; + + const U32 maxSearches = 1 << ctx->params.cParams.searchLog; + const U32 mls = ctx->params.cParams.searchLength; + + /* init */ + U32 rep[ZSTD_REP_INIT]; + for (U32 i=0; i= 3) /* intentional overflow */ + if (MEM_read32(start) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(start+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = ZSTD_HcFindBestMatch_extDict_selectMLS(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < EQUAL_READ32) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* catch up */ + if (offset >= ZSTD_REP_NUM) { + U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } + rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE); + } + +_storeSequence: + /* store sequence */ + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repIndex = (U32)((ip-base) - rep[1]); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; } } -FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( - ZSTD_CCtx* zc, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 maxNbAttempts, const U32 matchLengthSearch) -{ - switch(matchLengthSearch) - { - default : - case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); - case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); - case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); - } -} - /* ******************************* * Common parser - lazy strategy @@ -1632,9 +1894,9 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, const BYTE* const ilimit = iend - 8; const BYTE* const base = ctx->base + ctx->dictLimit; - size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE; const U32 maxSearches = 1 << ctx->params.cParams.searchLog; const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (mls == 3) ? 3 : 4; typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, @@ -1642,6 +1904,11 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; /* init */ + U32 rep[ZSTD_REP_INIT]; + for (U32 i=0; inextToUpdate3 = ctx->nextToUpdate; ZSTD_resetSeqStore(seqStorePtr); if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE; @@ -1649,13 +1916,22 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, while (ip < ilimit) { size_t matchLength=0; size_t offset=0; - const BYTE* start=ip+1; + const BYTE* start=ip; /* check repCode */ - if (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1)) { + for (U32 i=0; i gain1) + matchLength = mlRep, offset = i; + } } /* first search (depth 0) */ @@ -1674,12 +1950,13 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if (depth>=1) while (ip= MINMATCH) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; + matchLength = mlRep, offset = i, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); @@ -1693,12 +1970,13 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, /* let's find an even better one */ if ((depth==2) && (ip= MINMATCH) && (gain2 > gain1)) - matchLength = ml2, offset = 0, start = ip; + matchLength = ml2, offset = i, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); @@ -1712,32 +1990,34 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, } /* catch up */ - if (offset) { - while ((start>anchor) && (start>base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */ + if (offset >= ZSTD_REP_NUM) { + while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ { start--; matchLength++; } - offset_2 = offset_1; offset_1 = offset; } /* store sequence */ -_storeSequence: - { size_t const litLength = start - anchor; + { + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = (U32)(offset - ZSTD_REP_MOVE); + } else { + if (offset != 0) { + U32 temp = rep[offset]; + if (offset > 1) rep[2] = rep[1]; + if (offset > 0) rep[1] = rep[0]; + rep[0] = temp; + } + + if (offset<=1 && start==anchor) offset = 1-offset; + } + + size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } - - /* check immediate repcode */ - while ( (ip <= ilimit) - && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { - /* store sequence */ - matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend); - offset = offset_2; - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength); - ip += matchLength+MINMATCH; - anchor = ip; - continue; /* faster when present ... (?) */ - } } + } /* Last Literals */ { size_t const lastLLSize = iend - anchor; @@ -1748,9 +2028,13 @@ _storeSequence: } +/* The optimal parser */ +#include "zstd_opt.h" + + static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2); + ZSTD_compressBlock_opt_generic(ctx, src, srcSize); } static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) @@ -1770,7 +2054,7 @@ static void ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcS static void ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); + ZSTD_compressBlock_greedy_generic(ctx, src, srcSize); } @@ -1792,9 +2076,9 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const dictStart = dictBase + ctx->lowLimit; - size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE; const U32 maxSearches = 1 << ctx->params.cParams.searchLog; const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (mls == 3) ? 3 : 4; typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, @@ -1802,6 +2086,11 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; /* init */ + U32 rep[ZSTD_REP_INIT]; + for (U32 i=0; inextToUpdate3 = ctx->nextToUpdate; ZSTD_resetSeqStore(seqStorePtr); if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; @@ -1809,20 +2098,28 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, while (ip < ilimit) { size_t matchLength=0; size_t offset=0; - const BYTE* start=ip+1; + const BYTE* start=ip; U32 current = (U32)(ip-base); /* check repCode */ - { - const U32 repIndex = (U32)(current+1 - offset_1); + for (U32 i=0; i= 3) /* intentional overflow */ - if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; - if (depth==0) goto _storeSequence; + if (matchLength==0) { + offset = i; + matchLength = ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + } else { + size_t mlRep = ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + int gain2 = (int)(mlRep * 3 /*- ZSTD_highbit((U32)i+1)*/ + (i==1)); + int gain1 = (int)(matchLength*3 - /*ZSTD_highbit((U32)offset+1)*/ + 1 + (offset==1)); + if (gain2 > gain1) + matchLength = mlRep, offset = i; + } } } /* first search (depth 0) */ @@ -1843,19 +2140,19 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, ip ++; current++; /* check repCode */ - if (offset) { - const U32 repIndex = (U32)(current - offset_1); + for (U32 i=0; i= 3) /* intentional overflow */ - if (MEM_read32(ip) == MEM_read32(repMatch)) { + if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t const repLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; + size_t const repLength = ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1 + (offset= MINMATCH) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; + matchLength = repLength, offset = i, start = ip; } } /* search match, depth 1 */ @@ -1873,19 +2170,19 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, ip ++; current++; /* check repCode */ - if (offset) { - const U32 repIndex = (U32)(current - offset_1); + for (U32 i=0; i= 3) /* intentional overflow */ - if (MEM_read32(ip) == MEM_read32(repMatch)) { + if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t repLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; - int gain2 = (int)(repLength * 4); - int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); + size_t const repLength = ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1 + (offset= MINMATCH) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; + matchLength = repLength, offset = i, start = ip; } } /* search match, depth 2 */ @@ -1901,38 +2198,33 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, } /* catch up */ - if (offset) { - U32 matchIndex = (U32)((start-base) - offset); + if (offset >= ZSTD_REP_NUM) { + U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = offset; } /* store sequence */ -_storeSequence: - { size_t const litLength = start - anchor; + { + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = (U32)(offset - ZSTD_REP_MOVE); + } else { + if (offset != 0) { + U32 temp = rep[offset]; + if (offset > 1) rep[2] = rep[1]; + if (offset > 0) rep[1] = rep[0]; + rep[0] = temp; + } + + if (offset<=1 && start==anchor) offset = 1-offset; + } + + size_t const litLength = start - anchor; ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); anchor = ip = start + matchLength; - } - - /* check immediate repcode */ - while (ip <= ilimit) { - const U32 repIndex = (U32)((ip-base) - offset_2); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - if (MEM_read32(ip) == MEM_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; - offset = offset_2; offset_2 = offset_1; offset_1 = offset; /* swap offset history */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - break; } } /* Last Literals */ @@ -1944,7 +2236,7 @@ _storeSequence: void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); + ZSTD_compressBlock_greedy_extDict_generic(ctx, src, srcSize); } static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) @@ -1964,7 +2256,7 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 2); + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize); } @@ -1973,7 +2265,11 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) { static const ZSTD_blockCompressor blockCompressor[2][6] = { +#if 1 { ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt }, +#else + { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }, +#endif { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict } }; @@ -2116,8 +2412,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); - zc->params.cParams.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */ - ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.cParams.searchLength); return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); } @@ -2244,7 +2539,6 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, { BYTE* const op = (BYTE*)zc->headerBuffer; U32 const fcsId = (pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256); /* 0-3 */ BYTE fdescriptor = (BYTE)(params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); /* windowLog : 4 KB - 128 MB */ - fdescriptor |= (BYTE)((params.cParams.searchLength==3)<<4); /* mml : 3-4 */ fdescriptor |= (BYTE)(fcsId << 6); op[4] = fdescriptor; switch(fcsId) @@ -2505,12 +2799,21 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 14, 14, 14, 1, 4, 4, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 4, 4, ZSTD_fast }, /* level 2 */ { 14, 14, 14, 4, 4, 4, ZSTD_greedy }, /* level 3.*/ +#if 0 + { 14, 14, 14, 3, 3, 4, ZSTD_lazy }, /* level 4.*/ + { 14, 14, 14, 4, 3, 4, ZSTD_lazy2 }, /* level 5 */ + { 14, 14, 14, 5, 3, 4, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 3, 4, ZSTD_lazy2 }, /* level 7.*/ + { 14, 14, 14, 7, 3, 4, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 6, 3, 4, ZSTD_btlazy2 }, /* level 9.*/ +#else { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 4.*/ { 14, 14, 14, 4, 4, 4, ZSTD_lazy2 }, /* level 5 */ { 14, 14, 14, 5, 4, 4, ZSTD_lazy2 }, /* level 6 */ { 14, 14, 14, 6, 4, 4, ZSTD_lazy2 }, /* level 7.*/ { 14, 14, 14, 7, 4, 4, ZSTD_lazy2 }, /* level 8.*/ { 14, 15, 14, 6, 4, 4, ZSTD_btlazy2 }, /* level 9.*/ +#endif { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 436058b52..2dbe5ce89 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -148,8 +148,6 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; - dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->fParams.mml); return 0; } @@ -306,7 +304,6 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t memset(fparamsPtr, 0, sizeof(*fparamsPtr)); { BYTE const frameDesc = ip[4]; fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH; if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ switch(frameDesc >> 6) /* fcsId */ { @@ -621,11 +618,12 @@ typedef struct { FSE_DState_t stateLL; FSE_DState_t stateOffb; FSE_DState_t stateML; - size_t prevOffset; + size_t prevOffset[ZSTD_REP_INIT]; } seqState_t; -static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) + +static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) { /* Literal length */ U32 const llCode = FSE_peekSymbol(&(seqState->stateLL)); @@ -655,14 +653,38 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 }; /* sequence */ - { size_t const offset = ofCode ? OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits) : /* <= 26 bits */ - llCode ? seq->offset : seqState->prevOffset; - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); - if (ofCode | !llCode) seqState->prevOffset = seq->offset; /* cmove */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */ + if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); + } + + if (offset < ZSTD_REP_NUM) { + if (llCode == 0 && offset <= 1) offset = 1-offset; + + if (offset != 0) { + size_t temp = seqState->prevOffset[offset]; + if (offset != 1) { + seqState->prevOffset[2] = seqState->prevOffset[1]; + } + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + + } else { + offset = seqState->prevOffset[0]; + } + } else { + offset -= ZSTD_REP_MOVE; + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } seq->offset = offset; } - seq->matchLength = ML_base[mlCode] + mls + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ @@ -734,7 +756,7 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, } op += 8; match += 8; - if (oMatchEnd > oend-(16-3)) { // 3 = MINMATCH + if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_8) { ZSTD_wildcopy(op, match, oend_8 - op); match += oend_8 - op; @@ -767,7 +789,6 @@ static size_t ZSTD_decompressSequences( const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - const U32 mls = dctx->fParams.mml; int nbSeq; /* Build Decoding Tables */ @@ -785,7 +806,8 @@ static size_t ZSTD_decompressSequences( memset(&sequence, 0, sizeof(sequence)); sequence.offset = REPCODE_STARTVALUE; - seqState.prevOffset = REPCODE_STARTVALUE; + for (U32 i=0; i 200802300) && (pos < 200802400)) + // if ((pos > 200802300) && (pos < 200802400)) printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset); } @@ -843,8 +865,6 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->fParams.mml); - /* Decode literals sub-block */ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); if (ZSTD_isError(litCSize)) return litCSize; @@ -951,7 +971,6 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->fParams.mml); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index ce36aad3b..34d8b5441 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -65,6 +65,10 @@ #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A435 +#define ZSTD_REP_NUM 3 +#define ZSTD_REP_INIT ZSTD_REP_NUM +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) + #define KB *(1 <<10) #define MB *(1 <<20) #define GB *(1U<<30) @@ -95,7 +99,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define LONGNBSEQ 0x7F00 -#define MINMATCH 4 +#define MINMATCH 3 +#define EQUAL_READ32 4 #define REPCODE_STARTVALUE 1 #define Litbits 8 @@ -188,8 +193,7 @@ typedef struct { U32 off; U32 mlen; U32 litlen; - U32 rep; - U32 rep2; + U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; #if ZSTD_OPT_DEBUG == 3 diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 145570031..2c7df9f59 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -215,26 +215,6 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B /*-************************************* * Binary Tree search ***************************************/ -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (ie. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) -{ - U32* const hashTable3 = zc->hashTable3; - U32 const hashLog3 = zc->hashLog3; - const BYTE* const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate3; - - while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; - idx++; - } - - zc->nextToUpdate3 = target; - return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)]; -} - - static U32 ZSTD_insertBtAndGetAllMatches ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, @@ -267,8 +247,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( size_t bestLength = minMatch-1; if (minMatch == 3) { /* HC3 match finder */ - U32 matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); - + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { const BYTE* match; size_t currentMl=0; @@ -277,14 +256,14 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); } else { match = dictBase + matchIndex3; - if (MEM_readMINMATCH(match, minMatch) == MEM_readMINMATCH(ip, minMatch)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+minMatch, match+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; } /* save best solution */ if (currentMl > bestLength) { bestLength = currentMl; - matches[mnum].off = current - matchIndex3; + matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3; matches[mnum].len = (U32)currentMl; mnum++; if (currentMl > ZSTD_OPT_NUM) goto update; @@ -329,7 +308,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = current - matchIndex; + matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex; matches[mnum].len = (U32)matchLength; mnum++; if (matchLength > ZSTD_OPT_NUM) break; @@ -421,8 +400,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( *********************************/ FORCE_INLINE void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 depth) + const void* src, size_t srcSize) { seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; @@ -434,7 +412,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, const BYTE* const base = ctx->base; const BYTE* const prefixStart = base + ctx->dictLimit; - U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE; const U32 maxSearches = 1U << ctx->params.cParams.searchLog; const U32 sufficient_len = ctx->params.cParams.targetLength; const U32 mls = ctx->params.cParams.searchLength; @@ -446,6 +423,10 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, U32 cur, match_num, last_pos, litlen, price; /* init */ + U32 rep[ZSTD_REP_INIT]; + for (U32 i=0; inextToUpdate3 = ctx->nextToUpdate; ZSTD_resetSeqStore(seqStorePtr); ZSTD_rescaleFreqs(seqStorePtr); @@ -466,21 +447,23 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, opt[0].litlen = (U32)(ip - litstart); /* check repCode */ - if (MEM_readMINMATCH(ip+1, minMatch) == MEM_readMINMATCH(ip+1 - rep_1, minMatch)) { + for (U32 i=0; i sufficient_len || mlen >= ZSTD_OPT_NUM) { - ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; + ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; } - litlen = opt[0].litlen + 1; + best_off = (i<=1 && ip == anchor) ? 1-i : i; + litlen = opt[0].litlen; do { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); - if (mlen + 1 > last_pos || price < opt[mlen + 1].price) - SET_PRICE(mlen + 1, mlen, 0, litlen, price); /* note : macro modifies last_pos */ + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ mlen--; } while (mlen >= minMatch); } @@ -490,8 +473,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } - opt[0].rep = rep_1; - opt[0].rep2 = rep_2; + for (U32 i=0; i sufficient_len) { @@ -511,7 +494,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; @@ -521,7 +504,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { - size_t cur_rep; inr = ip + cur; if (opt[cur-1].mlen == 1) { @@ -544,67 +526,55 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - - if (opt[cur].off) { - opt[cur].rep2 = opt[cur-mlen].rep; - opt[cur].rep = opt[cur].off; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); + if (opt[cur].off >= ZSTD_REP_NUM) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { - if (cur!=mlen && opt[cur].litlen == 0) { - opt[cur].rep2 = opt[cur-mlen].rep; - opt[cur].rep = opt[cur-mlen].rep2; - ZSTD_LOG_ENCODE("%d: COPYREP_SWI cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } else { - opt[cur].rep2 = opt[cur-mlen].rep2; - opt[cur].rep = opt[cur-mlen].rep; - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } } - - ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); - - best_mlen = 0; - - if (opt[cur].mlen != 1) { - cur_rep = opt[cur].rep2; - ZSTD_LOG_PARSER("%d: tryNoExt REP2 rep2=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); - } else { - cur_rep = opt[cur].rep; - ZSTD_LOG_PARSER("%d: tryNoExt REP1 rep=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } - if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - cur_rep, minMatch)) { // check rep - mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - cur_rep, iend) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); + ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + best_mlen = 0; + + for (U32 i=0; i sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = 0; ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); - last_pos = cur + 1; + best_mlen = mlen; best_off = i; last_pos = cur + 1; goto _storeSequence; } + best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); } else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); } best_mlen = mlen; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, 0, price, litlen); + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); do { if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, 0, litlen, price); + SET_PRICE(cur + mlen, mlen, i, litlen, price); mlen--; } while (mlen >= minMatch); } + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); @@ -623,21 +593,19 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); - while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); @@ -651,8 +619,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* store sequence */ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ for (u = 1; u <= last_pos; u++) - ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep); + ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); + ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); opt[0].mlen = 1; U32 offset; @@ -669,38 +637,42 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ } for (u = 0; u <= last_pos;) { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); + ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); u += opt[u].mlen; } for (cur=0; cur < last_pos; ) { - ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); + ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - U32 litLength = (U32)(ip - anchor); - ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); + U32 const litLength = (U32)(ip - anchor); + // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset) { - rep_2 = rep_1; - rep_1 = offset; + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE; } else { - if (litLength == 0) { - best_off = rep_2; - rep_2 = rep_1; - rep_1 = best_off; - } } + if (offset != 0) { + best_off = rep[offset]; + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength == 0 && offset<=1) offset = 1-offset; + } - // ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); + ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset) - ml2 = (U32)ZSTD_count(ip, ip-offset, iend); + if (offset >= ZSTD_REP_NUM) + ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend); else - ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); + ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { @@ -709,27 +681,10 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } #endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-minMatch); + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; - } /* for (cur=0; cur < last_pos; ) */ - - /* check immediate repcode */ - while ((anchor >= prefixStart + rep_2) && (anchor <= ilimit) - && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(anchor - rep_2, minMatch)) ) { - /* store sequence */ - best_mlen = (U32)ZSTD_count(anchor+minMatch, anchor+minMatch-rep_2, iend); - best_off = rep_2; - rep_2 = rep_1; - rep_1 = best_off; - ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, best_mlen); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, best_mlen); - anchor += best_mlen+minMatch; - continue; /* faster when present ... (?) */ - } - if (anchor > ip) ip = anchor; - } + } } /* for (cur=0; cur < last_pos; ) */ { /* Last Literals */ size_t lastLLSize = iend - anchor; @@ -742,8 +697,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ FORCE_INLINE void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize, - const U32 depth) + const void* src, size_t srcSize) { seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; @@ -757,9 +711,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictBase = ctx->dictBase; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = ctx->lowLimit; - U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE; const U32 maxSearches = 1U << ctx->params.cParams.searchLog; const U32 sufficient_len = ctx->params.cParams.targetLength; const U32 mls = ctx->params.cParams.searchLength; @@ -771,6 +723,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, U32 cur, match_num, last_pos, litlen, price; /* init */ + U32 rep[ZSTD_REP_INIT]; + for (U32 i=0; inextToUpdate3 = ctx->nextToUpdate; ZSTD_resetSeqStore(seqStorePtr); ZSTD_rescaleFreqs(seqStorePtr); @@ -780,8 +736,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* Match Loop */ while (ip < ilimit) { - U32 u, offset, best_off=0; - U32 mlen=0, best_mlen=0; + U32 u; + U32 mlen=0; + U32 best_mlen=0; + U32 best_off=0; U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; @@ -790,49 +748,50 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[0].litlen = (U32)(ip - litstart); /* check repCode */ - { - const U32 repIndex = (U32)(current+1 - rep_1); + for (U32 i=0; i= 3) /* intentional overflow */ - && (MEM_readMINMATCH(ip+1, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(ip+1+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen); - if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; + ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; } - litlen = opt[0].litlen + 1; + best_off = (i<=1 && ip == anchor) ? 1-i : i; + litlen = opt[0].litlen; do { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); - if (mlen + 1 > last_pos || price < opt[mlen + 1].price) - SET_PRICE(mlen + 1, mlen, 0, litlen, price); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ mlen--; } while (mlen >= minMatch); } } - best_mlen = (last_pos) ? last_pos : minMatch; + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } - ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); - if (!last_pos && !match_num) { ip++; continue; } + for (U32 i=0; i sufficient_len) { + if (match_num && matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; last_pos = 1; goto _storeSequence; - } + } + + best_mlen = (last_pos) ? last_pos : minMatch; // set prices using matches at position = 0 for (u = 0; u < match_num; u++) { @@ -841,7 +800,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; @@ -854,98 +813,81 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { - size_t cur_rep; - inr = ip + cur; + inr = ip + cur; - if (opt[cur-1].mlen == 1) { + if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); } else price = ZSTD_getLiteralPrice(seqStorePtr, litlen, litstart); - } else { + } else { litlen = 1; price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); - } - - if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) break; - - if (inr > ilimit) // last match must start at a minimum distance of 8 from oend - continue; - - mlen = opt[cur].mlen; - - if (opt[cur].off) { - opt[cur].rep2 = opt[cur-mlen].rep; - opt[cur].rep = opt[cur].off; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } else { - if (cur!=mlen && opt[cur].litlen == 0) { - opt[cur].rep2 = opt[cur-mlen].rep; - opt[cur].rep = opt[cur-mlen].rep2; - ZSTD_LOG_ENCODE("%d: COPYREP_SWI cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } else { - opt[cur].rep2 = opt[cur-mlen].rep2; - opt[cur].rep = opt[cur-mlen].rep; - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } } - - ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); - - best_mlen = 0; - - if (opt[cur].mlen != 1) { - cur_rep = opt[cur].rep2; - ZSTD_LOG_PARSER("%d: tryExt REP2 rep2=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); - } else { - cur_rep = opt[cur].rep; - ZSTD_LOG_PARSER("%d: tryExt REP1 rep=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); - } - - const U32 repIndex = (U32)(current+cur - cur_rep); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { - /* repcode detected */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; - best_off = 0; - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); - last_pos = cur + 1; - goto _storeSequence; - } - - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - minMatch); - } - - best_mlen = mlen; - - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, 0, price, litlen); - - do { - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH - SET_PRICE(cur + mlen, mlen, 0, litlen, price); - mlen--; - } while (mlen >= minMatch); } - best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; + if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off >= ZSTD_REP_NUM) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } + + ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + best_mlen = 0; + + for (U32 i=0; i= 3) /* intentional overflow */ + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); + } + + best_mlen = mlen; + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); + + do { + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); @@ -957,44 +899,45 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, goto _storeSequence; } - // set prices using matches at position = cur + best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; + + /* set prices using matches at position = cur */ for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); - while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH); } else { litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); mlen++; - } } } // for (cur = 1; cur <= last_pos; cur++) + } } } /* for (cur = 1; cur <= last_pos; cur++) */ best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; /* store sequence */ -_storeSequence: // cur, last_pos, best_mlen, best_off have to be set +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ for (u = 1; u <= last_pos; u++) - ZSTD_LOG_PARSER("%d: price[%u/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep); + ZSTD_LOG_PARSER("%d: price[%u/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); + ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); opt[0].mlen = 1; + U32 offset; while (1) { mlen = opt[cur].mlen; @@ -1008,44 +951,48 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } for (u = 0; u <= last_pos; ) { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); + ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); u += opt[u].mlen; } for (cur=0; cur < last_pos; ) { - U32 litLength; - ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); + ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - litLength = (U32)(ip - anchor); - ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); + U32 const litLength = (U32)(ip - anchor); + // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - if (offset) { - rep_2 = rep_1; - rep_1 = offset; + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE; } else { - if (litLength == 0) { - best_off = rep_2; - rep_2 = rep_1; - rep_1 = best_off; - } } + if (offset != 0) { + best_off = rep[offset]; + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength == 0 && offset<=1) offset = 1-offset; + } - ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); + ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); #if ZSTD_OPT_DEBUG >= 5 U32 ml2; - if (offset) { - if (offset > (size_t)(ip - prefixStart)) { - const BYTE* match = dictEnd - (offset - (ip - prefixStart)); + if (offset >= ZSTD_REP_NUM) { + best_off = offset - ZSTD_REP_MOVE; + if (best_off > (size_t)(ip - prefixStart)) { + const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); - ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)offset, dictBase, dictEnd, prefixStart, ip, match); + ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); } else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); } - else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); + else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { @@ -1054,57 +1001,10 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } #endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-minMatch); + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; - } - -#if 0 - /* check immediate repcode */ - while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) { - if ((anchor - rep_2) >= prefixStart) { - if (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(anchor - rep_2, minMatch)) - mlen = (U32)ZSTD_count(anchor+minMatch, anchor - rep_2 + minMatch, iend) + minMatch; - else - break; - } else { - const BYTE* repMatch = dictBase + ((anchor-base) - rep_2); - if ((repMatch + minMatch <= dictEnd) && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch))) - mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, dictEnd, prefixStart) + minMatch; - else - break; - } - - offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ - ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); - anchor += mlen; - } -#else - /* check immediate repcode */ - /* minimal correctness condition = while ((anchor >= prefixStart + REPCODE_STARTVALUE) && (anchor <= ilimit)) { */ - while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) { - const U32 repIndex = (U32)((anchor-base) - rep_2); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { - /* repcode detected, let's take it */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ - ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); - anchor += mlen; - continue; /* faster when present ... (?) */ - } - break; - } -#endif - if (anchor > ip) ip = anchor; - } + } } /* for (cur=0; cur < last_pos; ) */ { /* Last Literals */ size_t lastLLSize = iend - anchor; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index bbc20240d..e804a5083 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -179,7 +179,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse ZSTD_CCtx to compress some new frame. */ -typedef struct { U64 frameContentSize; U32 windowLog; U32 mml; } ZSTD_frameParams; +typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams; #define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ static const size_t ZSTD_frameHeaderSize_min = 5; diff --git a/programs/Makefile b/programs/Makefile index 6aca9768e..47621caa7 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -53,7 +53,7 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -ZSTD_FILES := $(ZSTDDIR)/huff0.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c +ZSTD_FILES := $(ZSTDDIR)/huff0.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_compress.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 @@ -84,6 +84,10 @@ default: zstd all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen +zstd_clang: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ + zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c + clang.exe --target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zbuff.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)