From 4ab9c913a585da4f703a859cd1a7aecef7f8958f Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 4 Mar 2016 19:17:31 +0100 Subject: [PATCH 1/4] MEM_readMINMATCH replaced with inline function --- lib/mem.h | 10 +++ lib/zstd_compress.c | 35 ++++----- lib/zstd_opt.h | 161 ++++++++++++++++++++-------------------- lib/zstd_opt_internal.h | 41 ++-------- 4 files changed, 114 insertions(+), 133 deletions(-) diff --git a/lib/mem.h b/lib/mem.h index 0e357e530..a369033b1 100644 --- a/lib/mem.h +++ b/lib/mem.h @@ -188,6 +188,16 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : return MEM_read32(memPtr)<<8; + } +} + MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 50fa2fdc2..637ffb982 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -207,15 +207,16 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize; zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2); zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); + if (params.strategy == ZSTD_btopt) { + zc->seqStore.litFreq = (U32*)((void*)(zc->seqStore.dumpsStart + (blockSize>>2))); + zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (1<seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1)); + zc->seqStore.litLengthSum = 0; + } - zc->seqStore.litFreq = (U32*)((void*)(zc->seqStore.dumpsStart + (blockSize>>2))); - zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<seqStore.matchTable = (ZSTD_match_t*)(zc->seqStore.offCodeFreq + (1<seqStore.priceTable = (ZSTD_optimal_t*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1); - - zc->seqStore.litLengthSum = 0; zc->hbSize = 0; zc->stage = 0; zc->loadedDictEnd = 0; @@ -1661,10 +1662,7 @@ _storeSequence: static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - if (ctx->params.searchLength == 3) - ZSTD_compressBlock_opt_generic3(ctx, src, srcSize, 2); - else - ZSTD_compressBlock_opt_generic4(ctx, src, srcSize, 2); + ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 2); } static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) @@ -1883,10 +1881,7 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - if (ctx->params.searchLength == 3) - ZSTD_compressBlock_opt_extDict_generic3(ctx, src, srcSize, 2); - else - ZSTD_compressBlock_opt_extDict_generic4(ctx, src, srcSize, 2); + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 2); } @@ -2404,10 +2399,10 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 14, 15, 15, 0, 15, 4,256, ZSTD_btopt }, /* level 19 */ { 0, 14, 15, 15, 0, 16, 4,256, ZSTD_btopt }, /* level 20 */ { 0, 14, 15, 15, 0, 17, 4,256, ZSTD_btopt }, /* level 21 */ - { 0, 14, 15, 15, 0, 17, 4,256, ZSTD_btopt }, /* level 21-2 */ - { 0, 14, 15, 15, 0, 17, 4,256, ZSTD_btopt }, /* level 21-3 */ - { 0, 14, 15, 15, 0, 17, 4,256, ZSTD_btopt }, /* level 21-4 */ - { 0, 14, 15, 15, 0, 17, 4,256, ZSTD_btopt }, /* level 21-5 */ + { 0, 14, 15, 15, 0, 14, 3,256, ZSTD_btopt }, /* level 21-2 */ + { 0, 14, 15, 15, 0, 15, 3,256, ZSTD_btopt }, /* level 21-3 */ + { 0, 14, 15, 15, 0, 16, 3,256, ZSTD_btopt }, /* level 21-4 */ + { 0, 14, 15, 15, 0, 17, 3,256, ZSTD_btopt }, /* level 21-5 */ }, }; diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index f61485987..66700514f 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -41,7 +41,6 @@ FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYT U32 price = (offCode-1) + (!offCode) + ZSTD_highbit(seqStorePtr->offCodeSum+1) - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); /* match Length */ - matchLength -= MINMATCHOPT; price += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); if (matchLength >= MaxML) matchLength = MaxML; price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit(seqStorePtr->matchLengthSum+1) - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); @@ -95,36 +94,36 @@ static U32 ZSTD_INSERTBTANDGETALLMATCHES ( U32 dummy32; /* to be nullified at the end */ U32 mnum = 0; - size_t bestLength = MINMATCHOPT-1; + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t bestLength = minMatch-1; hashTable[h] = current; /* Update Hash Table */ -#if MINMATCHOPT == 3 - /* HC3 match finder */ - U32 matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (minMatch == 3) { /* HC3 match finder */ + U32 matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); - if (matchIndex3>windowLow) { - const BYTE* match; - size_t currentMl=0; - if ((!extDict) || matchIndex3 >= dictLimit) { - match = base + matchIndex3; - if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex3; - if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+MINMATCHOPT, match+MINMATCHOPT, iLimit, dictEnd, prefixStart) + MINMATCHOPT; - } + if (matchIndex3>windowLow) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex3; + if (MEM_readMINMATCH(match, minMatch) == MEM_readMINMATCH(ip, minMatch)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+minMatch, match+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } - /* save best solution */ - if (currentMl > bestLength) { - bestLength = currentMl; - matches[mnum].off = current - matchIndex3; - matches[mnum].len = (U32)currentMl; - mnum++; - if (currentMl > ZSTD_OPT_NUM) return mnum; - if (ip+currentMl == iLimit) return mnum; /* best possible, and avoid read overflow*/ + /* save best solution */ + if (currentMl > bestLength) { + bestLength = currentMl; + matches[mnum].off = current - matchIndex3; + matches[mnum].len = (U32)currentMl; + mnum++; + if (currentMl > ZSTD_OPT_NUM) return mnum; + if (ip+currentMl == iLimit) return mnum; /* best possible, and avoid read overflow*/ + } } } -#endif while (nbCompares-- && (matchIndex > windowLow)) { U32* nextPtr = bt + 2*(matchIndex & btMask); @@ -266,8 +265,9 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE; const U32 maxSearches = 1U << ctx->params.searchLog; - const U32 mls = ctx->params.searchLength; const U32 sufficient_len = ctx->params.targetLength; + const U32 mls = ctx->params.searchLength; + const U32 minMatch = (ctx->params.searchLength == 3) ? 3 : 4; ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; @@ -295,9 +295,9 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, opt[0].litlen = (U32)(ip - litstart); /* check repCode */ - if (MEM_readMINMATCH(ip+1) == MEM_readMINMATCH(ip+1 - rep_1)) { + if (MEM_readMINMATCH(ip+1, minMatch) == MEM_readMINMATCH(ip+1 - rep_1, minMatch)) { /* repcode : we take it */ - mlen = (U32)ZSTD_count(ip+1+MINMATCHOPT, ip+1+MINMATCHOPT-rep_1, iend) + MINMATCHOPT; + mlen = (U32)ZSTD_count(ip+1+minMatch, ip+1+minMatch-rep_1, iend) + minMatch; ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen); if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { @@ -307,11 +307,11 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, litlen = opt[0].litlen + 1; do { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); if (mlen + 1 > last_pos || price < opt[mlen + 1].price) SET_PRICE(mlen + 1, mlen, 0, litlen, price); /* note : macro modifies last_pos */ mlen--; - } while (mlen >= MINMATCHOPT); + } while (mlen >= minMatch); } match_num = ZSTD_BTGETALLMATCHES_SELECTMLS(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ @@ -331,7 +331,7 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, goto _storeSequence; } - best_mlen = (last_pos) ? last_pos : MINMATCHOPT; + best_mlen = (last_pos) ? last_pos : minMatch; // set prices using matches at position = 0 for (u = 0; u < match_num; u++) { @@ -340,13 +340,13 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; } } - if (last_pos < MINMATCHOPT) { ip++; continue; } + if (last_pos < minMatch) { ip++; continue; } /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { @@ -401,8 +401,8 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: tryNoExt REP1 rep=%u mlen=%u\n", (int)(inr-base), (U32)cur_rep, mlen); } - if (MEM_readMINMATCH(inr) == MEM_readMINMATCH(inr - cur_rep)) { // check rep - mlen = (U32)ZSTD_count(inr+MINMATCHOPT, inr+MINMATCHOPT - cur_rep, iend) + MINMATCHOPT; + if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - cur_rep, minMatch)) { // check rep + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - cur_rep, iend) + minMatch; ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { @@ -416,12 +416,12 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, inr-litlen, 0, mlen); + price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); } else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, 0, mlen); + price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, 0, mlen - minMatch); } best_mlen = mlen; @@ -431,7 +431,7 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, if (cur + mlen > last_pos || price <= opt[cur + mlen].price) SET_PRICE(cur + mlen, mlen, 0, litlen, price); mlen--; - } while (mlen >= MINMATCHOPT); + } while (mlen >= minMatch); } match_num = ZSTD_BTGETALLMATCHES_SELECTMLS(ctx, inr, ip, iend, maxSearches, mls, matches); @@ -444,7 +444,7 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, goto _storeSequence; } - best_mlen = (best_mlen > MINMATCHOPT) ? best_mlen : MINMATCHOPT; + best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; /* set prices using matches at position = cur */ for (u = 0; u < match_num; u++) { @@ -457,12 +457,12 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen); + price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, matches[u].off, mlen); + price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); @@ -530,7 +530,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ml2 = (U32)ZSTD_count(ip, ip-offset, iend); else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < MINMATCHOPT)) { + if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } @@ -538,23 +538,23 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } #endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCHOPT); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCHOPT); + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-minMatch); anchor = ip = ip + mlen; } /* for (cur=0; cur < last_pos; ) */ /* check immediate repcode */ while ((anchor >= prefixStart + rep_2) && (anchor <= ilimit) - && (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(anchor - rep_2)) ) { + && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(anchor - rep_2, minMatch)) ) { /* store sequence */ - best_mlen = (U32)ZSTD_count(anchor+MINMATCHOPT, anchor+MINMATCHOPT-rep_2, iend); + best_mlen = (U32)ZSTD_count(anchor+minMatch, anchor+minMatch-rep_2, iend); best_off = rep_2; rep_2 = rep_1; rep_1 = best_off; ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, best_mlen); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, best_mlen); - anchor += best_mlen+MINMATCHOPT; + anchor += best_mlen+minMatch; continue; /* faster when present ... (?) */ } if (anchor > ip) ip = anchor; @@ -590,8 +590,9 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE; const U32 maxSearches = 1U << ctx->params.searchLog; - const U32 mls = ctx->params.searchLength; const U32 sufficient_len = ctx->params.targetLength; + const U32 mls = ctx->params.searchLength; + const U32 minMatch = (ctx->params.searchLength == 3) ? 3 : 4; ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; @@ -623,10 +624,10 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(ip+1) == MEM_readMINMATCH(repMatch)) ) { + && (MEM_readMINMATCH(ip+1, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(ip+1+MINMATCHOPT, repMatch+MINMATCHOPT, iend, repEnd, prefixStart) + MINMATCHOPT; + mlen = (U32)ZSTD_count_2segments(ip+1+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen); if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { @@ -636,14 +637,14 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, litlen = opt[0].litlen + 1; do { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); if (mlen + 1 > last_pos || price < opt[mlen + 1].price) SET_PRICE(mlen + 1, mlen, 0, litlen, price); mlen--; - } while (mlen >= MINMATCHOPT); + } while (mlen >= minMatch); } } - best_mlen = (last_pos) ? last_pos : MINMATCHOPT; + best_mlen = (last_pos) ? last_pos : minMatch; match_num = ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ @@ -669,13 +670,13 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; } } - if (last_pos < MINMATCHOPT) { + if (last_pos < minMatch) { // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ ip++; continue; } @@ -737,10 +738,10 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - &&(MEM_readMINMATCH(inr) == MEM_readMINMATCH(repMatch)) ) { + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(inr+MINMATCHOPT, repMatch+MINMATCHOPT, iend, repEnd, prefixStart) + MINMATCHOPT; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { @@ -754,12 +755,12 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, inr-litlen, 0, mlen); + price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); } else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, 0, mlen); + price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, 0, mlen - minMatch); } best_mlen = mlen; @@ -770,10 +771,10 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, if (cur + mlen > last_pos || price <= opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH SET_PRICE(cur + mlen, mlen, 0, litlen, price); mlen--; - } while (mlen >= MINMATCHOPT); + } while (mlen >= minMatch); } - best_mlen = (best_mlen > MINMATCHOPT) ? best_mlen : MINMATCHOPT; + best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; match_num = ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT(ctx, inr, ip, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); @@ -796,12 +797,12 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen); + price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen); + price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, matches[u].off, mlen); + price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); @@ -874,7 +875,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); } else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < MINMATCHOPT)) { + if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } if (ip < anchor) { printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } @@ -882,8 +883,8 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } #endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCHOPT); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCHOPT); + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-minMatch); anchor = ip = ip + mlen; } @@ -891,22 +892,22 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set /* check immediate repcode */ while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) { if ((anchor - rep_2) >= prefixStart) { - if (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(anchor - rep_2)) - mlen = (U32)ZSTD_count(anchor+MINMATCHOPT, anchor - rep_2 + MINMATCHOPT, iend) + MINMATCHOPT; + if (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(anchor - rep_2, minMatch)) + mlen = (U32)ZSTD_count(anchor+minMatch, anchor - rep_2 + minMatch, iend) + minMatch; else break; } else { const BYTE* repMatch = dictBase + ((anchor-base) - rep_2); - if ((repMatch + MINMATCHOPT <= dictEnd) && (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(repMatch))) - mlen = (U32)ZSTD_count_2segments(anchor+MINMATCHOPT, repMatch+MINMATCHOPT, iend, dictEnd, prefixStart) + MINMATCHOPT; + if ((repMatch + minMatch <= dictEnd) && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch))) + mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, dictEnd, prefixStart) + minMatch; else break; } offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-MINMATCHOPT); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-MINMATCHOPT); + ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); anchor += mlen; } #else @@ -917,14 +918,14 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(repMatch)) ) { + && (MEM_readMINMATCH(anchor, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected, let's take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(anchor+MINMATCHOPT, repMatch+MINMATCHOPT, iend, repEnd, prefixStart) + MINMATCHOPT; + mlen = (U32)ZSTD_count_2segments(anchor+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep_1, (int)rep_2); - ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-MINMATCHOPT); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-MINMATCHOPT); + ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); anchor += mlen; continue; /* faster when present ... (?) */ } diff --git a/lib/zstd_opt_internal.h b/lib/zstd_opt_internal.h index b980eab08..6dd5f5165 100644 --- a/lib/zstd_opt_internal.h +++ b/lib/zstd_opt_internal.h @@ -168,39 +168,14 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)]; } - -#define MINMATCHOPT 4 -#define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)) -#define ZSTD_GETPRICE ZSTD_getPrice4 -#define ZSTD_INSERTBTANDGETALLMATCHES ZSTD_insertBtAndGetAllMatches4 -#define ZSTD_BTGETALLMATCHES ZSTD_BtGetAllMatches4 -#define ZSTD_BTGETALLMATCHES_SELECTMLS ZSTD_BtGetAllMatches_selectMLS4 -#define ZSTD_BTGETALLMATCHES_EXTDICT ZSTD_BtGetAllMatches_extDict4 -#define ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ZSTD_BtGetAllMatches_selectMLS_extDict4 -#define ZSTD_COMPRESSBLOCK_OPT_GENERIC ZSTD_compressBlock_opt_generic4 -#define ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC ZSTD_compressBlock_opt_extDict_generic4 -#include "zstd_opt.h" -#undef MINMATCHOPT -#undef MEM_readMINMATCH -#undef ZSTD_GETPRICE -#undef ZSTD_INSERTBTANDGETALLMATCHES -#undef ZSTD_BTGETALLMATCHES -#undef ZSTD_BTGETALLMATCHES_SELECTMLS -#undef ZSTD_BTGETALLMATCHES_EXTDICT -#undef ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT -#undef ZSTD_COMPRESSBLOCK_OPT_GENERIC -#undef ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC - -#define MINMATCHOPT 3 -#define MEM_readMINMATCH(ptr) ((U32)(MEM_read32(ptr)<<8)) -#define ZSTD_GETPRICE ZSTD_getPrice3 -#define ZSTD_INSERTBTANDGETALLMATCHES ZSTD_insertBtAndGetAllMatches3 -#define ZSTD_BTGETALLMATCHES ZSTD_BtGetAllMatches3 -#define ZSTD_BTGETALLMATCHES_SELECTMLS ZSTD_BtGetAllMatches_selectMLS3 -#define ZSTD_BTGETALLMATCHES_EXTDICT ZSTD_BtGetAllMatches_extDict3 -#define ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ZSTD_BtGetAllMatches_selectMLS_extDict3 -#define ZSTD_COMPRESSBLOCK_OPT_GENERIC ZSTD_compressBlock_opt_generic3 -#define ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC ZSTD_compressBlock_opt_extDict_generic3 +#define ZSTD_GETPRICE ZSTD_getPrice +#define ZSTD_INSERTBTANDGETALLMATCHES ZSTD_insertBtAndGetAllMatches +#define ZSTD_BTGETALLMATCHES ZSTD_BtGetAllMatches +#define ZSTD_BTGETALLMATCHES_SELECTMLS ZSTD_BtGetAllMatches_selectMLS +#define ZSTD_BTGETALLMATCHES_EXTDICT ZSTD_BtGetAllMatches_extDict +#define ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ZSTD_BtGetAllMatches_selectMLS_extDict +#define ZSTD_COMPRESSBLOCK_OPT_GENERIC ZSTD_compressBlock_opt_generic +#define ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC ZSTD_compressBlock_opt_extDict_generic #include "zstd_opt.h" From 805d2a7465bd921cd5b2d0044875bb636ffc2952 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 4 Mar 2016 19:31:57 +0100 Subject: [PATCH 2/4] removed zstd_opt_internal.h --- lib/zstd_compress.c | 2 +- lib/zstd_opt.h | 209 +++++++++++++++++++++++++++++++++------- lib/zstd_opt_internal.h | 182 ---------------------------------- 3 files changed, 173 insertions(+), 220 deletions(-) delete mode 100644 lib/zstd_opt_internal.h diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 637ffb982..8f14ff036 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1380,7 +1380,7 @@ static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const B while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); } -#include "zstd_opt_internal.h" +#include "zstd_opt.h" /** Tree updater, providing best match */ static size_t ZSTD_BtFindBestMatch_extDict ( diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 66700514f..f6f9acecf 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -31,10 +31,83 @@ - Zstd source repository : https://www.zstd.net */ -/* Note : this file is intended to be included within zstd_opt_internal.h */ +/* Note : this file is intended to be included within zstd_compress.c */ -FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +#define ZSTD_FREQ_DIV 5 + +/*-************************************* +* Price functions for optimal parser +***************************************/ +MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) +{ + unsigned u; + + if (ssPtr->litLengthSum == 0) { + ssPtr->litSum = 2*(1<litLengthSum = 1*(1<matchLengthSum = 1*(1<offCodeSum = 1*(1<matchSum = 2*(1<litFreq[u] = 2; + for (u=0; u<=MaxLL; u++) + ssPtr->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + ssPtr->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + ssPtr->offCodeFreq[u] = 1; + } else { + ssPtr->matchLengthSum = 0; + ssPtr->litLengthSum = 0; + ssPtr->offCodeSum = 0; + ssPtr->matchSum = 0; + ssPtr->litSum = 0; + + for (u=0; u<=MaxLit; u++) { + ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litSum += ssPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) { + ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; + } + for (u=0; u<=MaxML; u++) { + ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; + ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); + } + for (u=0; u<=MaxOff; u++) { + ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; + } + } +} + + +FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals) +{ + U32 price, u; + + if (litLength == 0) + return ZSTD_highbit(seqStorePtr->litLengthSum+1) - ZSTD_highbit(seqStorePtr->litLengthFreq[0]+1); + + /* literals */ + price = litLength * ZSTD_highbit(seqStorePtr->litSum+1); + for (u=0; u < litLength; u++) + price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1); + + /* literal Length */ + price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); + if (litLength >= MaxLL) litLength = MaxLL; + price += ZSTD_highbit(seqStorePtr->litLengthSum+1) - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1); + + return price; +} + + +FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { /* offset */ BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; @@ -63,10 +136,72 @@ FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYT } +MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + U32 u; + + /* literals */ + seqStorePtr->litSum += litLength; + for (u=0; u < litLength; u++) + seqStorePtr->litFreq[literals[u]]++; + + /* literal Length */ + seqStorePtr->litLengthSum++; + if (litLength >= MaxLL) + seqStorePtr->litLengthFreq[MaxLL]++; + else + seqStorePtr->litLengthFreq[litLength]++; + + /* match offset */ + seqStorePtr->offCodeSum++; + BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; + seqStorePtr->offCodeFreq[offCode]++; + + /* match Length */ + seqStorePtr->matchLengthSum++; + if (matchLength >= MaxML) + seqStorePtr->matchLengthFreq[MaxML]++; + else + seqStorePtr->matchLengthFreq[matchLength]++; +} + + +#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = mlen_; \ + opt[pos].off = offset_; \ + opt[pos].litlen = litlen_; \ + opt[pos].price = price_; \ + ZSTD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-base), (int)pos, (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + } + + + /*-************************************* * Binary Tree search ***************************************/ -static U32 ZSTD_INSERTBTANDGETALLMATCHES ( +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) +{ + U32* const hashTable3 = zc->hashTable3; + const U32 hashLog3 = zc->params.hashLog3; + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate3; + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + zc->nextToUpdate3 = target; + return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)]; +} + + +static U32 ZSTD_insertBtAndGetAllMatches ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, U32 nbCompares, const U32 mls, @@ -191,18 +326,18 @@ static U32 ZSTD_INSERTBTANDGETALLMATCHES ( /** Tree updater, providing best match */ -static U32 ZSTD_BTGETALLMATCHES ( +static U32 ZSTD_BtGetAllMatches ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches) { if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_INSERTBTANDGETALLMATCHES(zc, ip, iLimit, maxNbAttempts, mls, 0, matches); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches); } -static U32 ZSTD_BTGETALLMATCHES_SELECTMLS ( +static U32 ZSTD_BtGetAllMatches_selectMLS ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches) @@ -211,25 +346,25 @@ static U32 ZSTD_BTGETALLMATCHES_SELECTMLS ( switch(matchLengthSearch) { default : - case 4 : return ZSTD_BTGETALLMATCHES(zc, ip, iHighLimit, maxNbAttempts, 4, matches); - case 5 : return ZSTD_BTGETALLMATCHES(zc, ip, iHighLimit, maxNbAttempts, 5, matches); - case 6 : return ZSTD_BTGETALLMATCHES(zc, ip, iHighLimit, maxNbAttempts, 6, matches); + case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches); + case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches); + case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches); } } /** Tree updater, providing best match */ -static U32 ZSTD_BTGETALLMATCHES_EXTDICT ( +static U32 ZSTD_BtGetAllMatches_extDict ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches) { if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_INSERTBTANDGETALLMATCHES(zc, ip, iLimit, maxNbAttempts, mls, 1, matches); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches); } -static U32 ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ( +static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches) @@ -238,9 +373,9 @@ static U32 ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ( switch(matchLengthSearch) { default : - case 4 : return ZSTD_BTGETALLMATCHES_EXTDICT(zc, ip, iHighLimit, maxNbAttempts, 4, matches); - case 5 : return ZSTD_BTGETALLMATCHES_EXTDICT(zc, ip, iHighLimit, maxNbAttempts, 5, matches); - case 6 : return ZSTD_BTGETALLMATCHES_EXTDICT(zc, ip, iHighLimit, maxNbAttempts, 6, matches); + case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches); + case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches); + case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches); } } @@ -249,7 +384,7 @@ static U32 ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ( * Optimal parser *********************************/ FORCE_INLINE -void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, +void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, const void* src, size_t srcSize, const U32 depth) { @@ -307,14 +442,14 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, litlen = opt[0].litlen + 1; do { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); if (mlen + 1 > last_pos || price < opt[mlen + 1].price) SET_PRICE(mlen + 1, mlen, 0, litlen, price); /* note : macro modifies last_pos */ mlen--; } while (mlen >= minMatch); } - match_num = ZSTD_BTGETALLMATCHES_SELECTMLS(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -340,7 +475,7 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; @@ -416,12 +551,12 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); } else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, 0, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - minMatch); } best_mlen = mlen; @@ -434,7 +569,7 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } - match_num = ZSTD_BTGETALLMATCHES_SELECTMLS(ctx, inr, ip, iend, maxSearches, mls, matches); + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, ip, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && matches[match_num-1].len > sufficient_len) { @@ -457,12 +592,12 @@ void ZSTD_COMPRESSBLOCK_OPT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); @@ -570,7 +705,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ FORCE_INLINE -void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, +void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const void* src, size_t srcSize, const U32 depth) { @@ -637,7 +772,7 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, litlen = opt[0].litlen + 1; do { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); if (mlen + 1 > last_pos || price < opt[mlen + 1].price) SET_PRICE(mlen + 1, mlen, 0, litlen, price); mlen--; @@ -646,7 +781,7 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, best_mlen = (last_pos) ? last_pos : minMatch; - match_num = ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -670,7 +805,7 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); if (mlen > last_pos || price < opt[mlen].price) SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; @@ -755,12 +890,12 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - minMatch); } else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, 0, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, 0, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, 0, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - minMatch); } best_mlen = mlen; @@ -776,7 +911,7 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; - match_num = ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT(ctx, inr, ip, iend, maxSearches, mls, matches); + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, ip, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && matches[match_num-1].len > sufficient_len) { @@ -797,12 +932,12 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx, if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_GETPRICE(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - minMatch); else - price = ZSTD_GETPRICE(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); + price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - minMatch); } else { litlen = 0; - price = opt[cur].price + ZSTD_GETPRICE(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - minMatch); } // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); diff --git a/lib/zstd_opt_internal.h b/lib/zstd_opt_internal.h deleted file mode 100644 index 6dd5f5165..000000000 --- a/lib/zstd_opt_internal.h +++ /dev/null @@ -1,182 +0,0 @@ -/* - zstd_opt_internal - common optimal parser functions to include - Header File for include - Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ - -/* Note : this file is intended to be included within zstd_compress.c */ - -#ifndef ZSTD_OPT_INTERNAL_H_MODULE -#define ZSTD_OPT_INTERNAL_H_MODULE - - -/*-******************************************* -* The optimal parser -*********************************************/ -/*- Constants -*/ -#define ZSTD_FREQ_DIV 5 - - -MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) -{ - unsigned u; - - if (ssPtr->litLengthSum == 0) { - ssPtr->litSum = 2*(1<litLengthSum = 1*(1<matchLengthSum = 1*(1<offCodeSum = 1*(1<matchSum = 2*(1<litFreq[u] = 2; - for (u=0; u<=MaxLL; u++) - ssPtr->litLengthFreq[u] = 1; - for (u=0; u<=MaxML; u++) - ssPtr->matchLengthFreq[u] = 1; - for (u=0; u<=MaxOff; u++) - ssPtr->offCodeFreq[u] = 1; - } else { - ssPtr->matchLengthSum = 0; - ssPtr->litLengthSum = 0; - ssPtr->offCodeSum = 0; - ssPtr->matchSum = 0; - ssPtr->litSum = 0; - - for (u=0; u<=MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->litSum += ssPtr->litFreq[u]; - } - for (u=0; u<=MaxLL; u++) { - ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; - } - for (u=0; u<=MaxML; u++) { - ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; - ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); - } - for (u=0; u<=MaxOff; u++) { - ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; - } - } -} - - -MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) -{ - U32 u; - - /* literals */ - seqStorePtr->litSum += litLength; - for (u=0; u < litLength; u++) - seqStorePtr->litFreq[literals[u]]++; - - /* literal Length */ - seqStorePtr->litLengthSum++; - if (litLength >= MaxLL) - seqStorePtr->litLengthFreq[MaxLL]++; - else - seqStorePtr->litLengthFreq[litLength]++; - - /* match offset */ - seqStorePtr->offCodeSum++; - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; - seqStorePtr->offCodeFreq[offCode]++; - - /* match Length */ - seqStorePtr->matchLengthSum++; - if (matchLength >= MaxML) - seqStorePtr->matchLengthFreq[MaxML]++; - else - seqStorePtr->matchLengthFreq[matchLength]++; -} - -FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals) -{ - U32 price, u; - - if (litLength == 0) - return ZSTD_highbit(seqStorePtr->litLengthSum+1) - ZSTD_highbit(seqStorePtr->litLengthFreq[0]+1); - - /* literals */ - price = litLength * ZSTD_highbit(seqStorePtr->litSum+1); - for (u=0; u < litLength; u++) - price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1); - - /* literal Length */ - price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); - if (litLength >= MaxLL) litLength = MaxLL; - price += ZSTD_highbit(seqStorePtr->litLengthSum+1) - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1); - - return price; -} - -#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ - { \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ - opt[pos].mlen = mlen_; \ - opt[pos].off = offset_; \ - opt[pos].litlen = litlen_; \ - opt[pos].price = price_; \ - ZSTD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-base), (int)pos, (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ - } - -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (ie. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) -{ - U32* const hashTable3 = zc->hashTable3; - const U32 hashLog3 = zc->params.hashLog3; - const BYTE* const base = zc->base; - const U32 target = (U32)(ip - base); - U32 idx = zc->nextToUpdate3; - - while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; - idx++; - } - - zc->nextToUpdate3 = target; - return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)]; -} - -#define ZSTD_GETPRICE ZSTD_getPrice -#define ZSTD_INSERTBTANDGETALLMATCHES ZSTD_insertBtAndGetAllMatches -#define ZSTD_BTGETALLMATCHES ZSTD_BtGetAllMatches -#define ZSTD_BTGETALLMATCHES_SELECTMLS ZSTD_BtGetAllMatches_selectMLS -#define ZSTD_BTGETALLMATCHES_EXTDICT ZSTD_BtGetAllMatches_extDict -#define ZSTD_BTGETALLMATCHES_SELECTMLS_EXTDICT ZSTD_BtGetAllMatches_selectMLS_extDict -#define ZSTD_COMPRESSBLOCK_OPT_GENERIC ZSTD_compressBlock_opt_generic -#define ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC ZSTD_compressBlock_opt_extDict_generic -#include "zstd_opt.h" - - -#endif /* ZSTD_OPT_INTERNAL_H_MODULE */ From e29caf7d1c5327ff7c2d585d55dec505f53ed325 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 4 Mar 2016 19:52:23 +0100 Subject: [PATCH 3/4] code cleanup --- lib/zstd_compress.c | 5 +++-- lib/zstd_internal.h | 4 ++-- lib/zstd_opt.h | 8 ++------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 8f14ff036..385b99dcb 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -731,7 +731,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n", (U32)(literals - g_start), (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif -#if ZSTD_OPT_DEBUG >= 3 +#if ZSTD_OPT_DEBUG == 3 if (offsetCode == 0) seqStorePtr->realRepSum++; seqStorePtr->realSeqSum++; seqStorePtr->realMatchSum += matchCode; @@ -1917,11 +1917,12 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.windowLog; +#if ZSTD_OPT_DEBUG == 3 seqStore_t* ssPtr = &zc->seqStore; static U32 priceFunc = 0; - ssPtr->realMatchSum = ssPtr->realLitSum = ssPtr->realSeqSum = ssPtr->realRepSum = 1; ssPtr->priceFunc = priceFunc; +#endif while (remaining) { size_t cSize; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 4948e2397..e136a89cc 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -206,13 +206,13 @@ typedef struct { U32 litLengthSum; U32 litSum; U32 offCodeSum; +#if ZSTD_OPT_DEBUG == 3 U32 realMatchSum; U32 realLitSum; U32 realSeqSum; U32 realRepSum; - U32 factor; - U32 factor2; U32 priceFunc; +#endif } seqStore_t; seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index f6f9acecf..780c5d415 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -118,16 +118,12 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT if (matchLength >= MaxML) matchLength = MaxML; price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit(seqStorePtr->matchLengthSum+1) - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); -#if ZSTD_OPT_DEBUG >= 3 +#if ZSTD_OPT_DEBUG == 3 switch (seqStorePtr->priceFunc) { default: case 0: - return 1 + price + seqStorePtr->factor + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); + return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); case 1: - return 1 + price + seqStorePtr->factor + ((seqStorePtr->factor2) ? ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)) : 0); - case 2: - return 1 + price + seqStorePtr->factor + ((seqStorePtr->factor2) ? ((seqStorePtr->litSum>>4) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)) : 0); - case 3: return 1 + price; } #else From f3c6503e5517a855f065428eca44971c1a0dd482 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 4 Mar 2016 20:04:25 +0100 Subject: [PATCH 4/4] resolve conflict in zstd_internal.h --- lib/zstd_internal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index e136a89cc..b06097a21 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -105,12 +105,13 @@ static const size_t ZSTD_frameHeaderSize_min = 5; #define OffFSELog 9 #define MaxSeq MAX(MaxLL, MaxML) +#define LONGNBSEQ 0xFF00 + #define FSE_ENCODING_RAW 0 #define FSE_ENCODING_RLE 1 #define FSE_ENCODING_STATIC 2 #define FSE_ENCODING_DYNAMIC 3 - #define HufLog 12 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */