From 93c3184d44e5cbef4ef48be29be7e063dc72a466 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 13 Jun 2018 16:54:31 -0400 Subject: [PATCH 01/22] Attach Dicts when Using ZSTD_btopt and ZSTD_btultra --- lib/compress/zstd_compress.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c071c5738..75338326d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1245,7 +1245,6 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN ) && !params.forceWindow /* dictMatchState isn't correctly * handled in _enforceMaxDist */ - && cdict->cParams.strategy <= ZSTD_btlazy2 && ZSTD_equivalentCParams(cctx->appliedParams.cParams, cdict->cParams); From d5d8240967eb5db5f7f67902854ab192610e7885 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 13 Jun 2018 17:10:02 -0400 Subject: [PATCH 02/22] Convert `extDict` Flag to `dictMode` Enum --- lib/compress/zstd_opt.c | 52 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 91ae4ae28..00e0e68e6 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -362,7 +362,7 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* static U32 ZSTD_insertBt1( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, - U32 const mls, U32 const extDict) + U32 const mls, const ZSTD_dictMode_e dictMode) { U32* const hashTable = ms->hashTable; U32 const hashLog = cParams->hashLog; @@ -425,8 +425,8 @@ static U32 ZSTD_insertBt1( } #endif - if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { - assert(matchIndex+matchLength >= dictLimit); /* might be wrong if extDict is incorrectly set to 0 */ + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); } else { @@ -472,16 +472,16 @@ FORCE_INLINE_TEMPLATE void ZSTD_updateTree_internal( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, - const U32 mls, const U32 extDict) + const U32 mls, const ZSTD_dictMode_e dictMode) { const BYTE* const base = ms->window.base; U32 const target = (U32)(ip - base); U32 idx = ms->nextToUpdate; - DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)", - idx, target, extDict); + DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); while(idx < target) - idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, extDict); + idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, dictMode); ms->nextToUpdate = target; } @@ -489,13 +489,13 @@ void ZSTD_updateTree( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* iend) { - ZSTD_updateTree_internal(ms, cParams, ip, iend, cParams->searchLength, 0 /*extDict*/); + ZSTD_updateTree_internal(ms, cParams, ip, iend, cParams->searchLength, ZSTD_noDict); } FORCE_INLINE_TEMPLATE U32 ZSTD_insertBtAndGetAllMatches ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* const ip, const BYTE* const iLimit, int const extDict, + const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, U32 rep[ZSTD_REP_NUM], U32 const ll0, ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) { @@ -542,7 +542,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( } else { /* repIndex < dictLimit || repIndex >= current */ const BYTE* const repMatch = dictBase + repIndex; assert(current >= windowLow); - if ( extDict /* this case only valid in extDict mode */ + if ( dictMode == ZSTD_extDict /* this case only valid in extDict mode */ && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { @@ -567,7 +567,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( if ((matchIndex3 > windowLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; - if ((!extDict) /*static*/ || (matchIndex3 >= dictLimit)) { + if ((dictMode != ZSTD_extDict) /*static*/ || (matchIndex3 >= dictLimit)) { const BYTE* const match = base + matchIndex3; mlen = ZSTD_count(ip, match, iLimit); } else { @@ -599,7 +599,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* match; assert(current > matchIndex); - if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); @@ -651,22 +651,22 @@ U32 ZSTD_insertBtAndGetAllMatches ( FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* const iHighLimit, int const extDict, + const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, U32 rep[ZSTD_REP_NUM], U32 const ll0, ZSTD_match_t* matches, U32 const lengthToBeat) { U32 const matchLengthSearch = cParams->searchLength; DEBUGLOG(8, "ZSTD_BtGetAllMatches"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, extDict); + ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, dictMode); switch(matchLengthSearch) { - case 3 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 3); + case 3 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3); default : - case 4 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 4); - case 5 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 5); + case 4 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4); + case 5 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5); case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, extDict, rep, ll0, matches, lengthToBeat, 6); + case 6 : return ZSTD_insertBtAndGetAllMatches(ms, cParams, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6); } } @@ -711,7 +711,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 rep[ZSTD_REP_NUM], const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize, - const int optLevel, const int extDict) + const int optLevel, const ZSTD_dictMode_e dictMode) { optState_t* const optStatePtr = &ms->opt; const BYTE* const istart = (const BYTE*)src; @@ -743,7 +743,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, ip, iend, extDict, rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, ip, iend, dictMode, rep, ll0, matches, minMatch); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -840,7 +840,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const previousPrice = opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, extDict, opt[cur].rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch); U32 matchNb; if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -973,7 +973,7 @@ size_t ZSTD_compressBlock_btopt( const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_btopt"); - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, 0 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); } @@ -1019,7 +1019,7 @@ size_t ZSTD_compressBlock_btultra( assert(ms->nextToUpdate >= ms->window.dictLimit && ms->nextToUpdate <= ms->window.dictLimit + 1); memcpy(tmpRep, rep, sizeof(tmpRep)); - ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); /* generate stats into ms->opt*/ + ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ ZSTD_resetSeqStore(seqStore); /* invalidate first scan from history */ ms->window.base -= srcSize; @@ -1031,19 +1031,19 @@ size_t ZSTD_compressBlock_btultra( ZSTD_upscaleStats(&ms->opt); } #endif - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); } size_t ZSTD_compressBlock_btopt_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, 1 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); } size_t ZSTD_compressBlock_btultra_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 1 /*extDict*/); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); } From ccbf067973e3510ebd52c711ce0e969cb04852cd Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 13 Jun 2018 17:10:37 -0400 Subject: [PATCH 03/22] Add _dictMatchState Functions --- lib/compress/zstd_compress.c | 3 ++- lib/compress/zstd_opt.c | 14 ++++++++++++++ lib/compress/zstd_opt.h | 7 +++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 75338326d..01ba01bda 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2254,7 +2254,8 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo ZSTD_compressBlock_lazy_dictMatchState, ZSTD_compressBlock_lazy2_dictMatchState, ZSTD_compressBlock_btlazy2_dictMatchState, - NULL, NULL /* unimplemented as of yet */ } + ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState } }; ZSTD_blockCompressor selectedCompressor; ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 00e0e68e6..aa0ed6627 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1034,6 +1034,20 @@ size_t ZSTD_compressBlock_btultra( return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); } +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState); +} + size_t ZSTD_compressBlock_btopt_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index b8dc389f3..63dbe79a8 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -28,6 +28,13 @@ size_t ZSTD_compressBlock_btultra( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); + size_t ZSTD_compressBlock_btopt_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); From 1e03377bde37d8f16136af97f3ef4b437bde806f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 13:23:17 -0400 Subject: [PATCH 04/22] Implement RepCode Check --- lib/compress/zstd_opt.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index aa0ed6627..1bb9bec20 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -524,6 +524,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 mnum = 0; U32 nbCompares = 1U << cParams->searchLog; + const ZSTD_matchState_t* dms = ms->dictMatchState; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + size_t bestLength = lengthToBeat-1; DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current); @@ -547,7 +554,14 @@ U32 ZSTD_insertBtAndGetAllMatches ( & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; - } } + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)) { + const BYTE* const repMatch = dmsBase + repIndex - dmsIndexDelta; + if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } } } /* save longer solution */ if (repLen > bestLength) { DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", From a075864756ebee79cb8969ee2ec8cf1e08e57103 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 13:33:59 -0400 Subject: [PATCH 05/22] Switch `!= ZSTD_extDict` to `== ZSTD_noDict` --- lib/compress/zstd_opt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 1bb9bec20..20a8ac2e0 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -425,7 +425,7 @@ static U32 ZSTD_insertBt1( } #endif - if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + if ((dictMode == ZSTD_noDict) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); @@ -581,7 +581,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( if ((matchIndex3 > windowLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; - if ((dictMode != ZSTD_extDict) /*static*/ || (matchIndex3 >= dictLimit)) { + if ((dictMode == ZSTD_noDict) /*static*/ || (matchIndex3 >= dictLimit)) { const BYTE* const match = base + matchIndex3; mlen = ZSTD_count(ip, match, iLimit); } else { @@ -613,7 +613,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* match; assert(current > matchIndex); - if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + if ((dictMode == ZSTD_noDict) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); From ce743312e2e91c35de209616dec550bf65b1849d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 14:52:46 -0400 Subject: [PATCH 06/22] Fix Typo --- lib/compress/zstd_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 20a8ac2e0..f638c150a 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -560,7 +560,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)) { const BYTE* const repMatch = dmsBase + repIndex - dmsIndexDelta; if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) { - repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; } } } /* save longer solution */ if (repLen > bestLength) { From ade8586ce6175d1cd2df9e3902d79197f3a309c4 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 14:53:04 -0400 Subject: [PATCH 07/22] Find `mls == 3` Matches --- lib/compress/zstd_opt.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index f638c150a..974802e17 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -603,7 +603,36 @@ U32 ZSTD_insertBtAndGetAllMatches ( (ip+mlen == iLimit) ) { /* best possible length */ ms->nextToUpdate = current+1; /* skip insertion */ return 1; - } } } } + } + } + } else if (dictMode == ZSTD_dictMatchState) { + /* should we perform this search even if the working ctx search + * found a match? */ + U32 const hashLog3 = ms->hashLog3; + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + U32 const dmsMatchIndex3 = dms->hashTable3[hash3]; + if (dmsMatchIndex3 >= dmsLowLimit) { + const BYTE* const match = dmsBase + dmsMatchIndex3; + size_t mlen = ZSTD_count_2segments(ip, match, iLimit, dmsEnd, prefixStart); + if (mlen >= mls) { + U32 const matchIndex3 = dmsMatchIndex3 + dmsIndexDelta; + DEBUGLOG(8, "found small dms match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(current > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = (current - matchIndex3) + ZSTD_REP_MOVE; + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = current+1; /* skip insertion */ + return 1; + } + } + } + } + } hashTable[h] = current; /* Update Hash Table */ From 64348a15f1327307d710d5fc3c05095befd26543 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 15:53:46 -0400 Subject: [PATCH 08/22] Misc Fixes --- lib/compress/zstd_opt.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 974802e17..8d90ee7ff 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -547,7 +547,9 @@ U32 ZSTD_insertBtAndGetAllMatches ( repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; } } else { /* repIndex < dictLimit || repIndex >= current */ - const BYTE* const repMatch = dictBase + repIndex; + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; assert(current >= windowLow); if ( dictMode == ZSTD_extDict /* this case only valid in extDict mode */ && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ @@ -557,11 +559,10 @@ U32 ZSTD_insertBtAndGetAllMatches ( } if (dictMode == ZSTD_dictMatchState && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ - & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)) { - const BYTE* const repMatch = dmsBase + repIndex - dmsIndexDelta; - if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) { - repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; - } } } + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } /* save longer solution */ if (repLen > bestLength) { DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", @@ -577,7 +578,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); + U32 matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); if ((matchIndex3 > windowLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; @@ -615,7 +616,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* const match = dmsBase + dmsMatchIndex3; size_t mlen = ZSTD_count_2segments(ip, match, iLimit, dmsEnd, prefixStart); if (mlen >= mls) { - U32 const matchIndex3 = dmsMatchIndex3 + dmsIndexDelta; + matchIndex3 = dmsMatchIndex3 + dmsIndexDelta; DEBUGLOG(8, "found small dms match with hlog3, of length %u", (U32)mlen); bestLength = mlen; @@ -642,7 +643,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* match; assert(current > matchIndex); - if ((dictMode == ZSTD_noDict) || (matchIndex+matchLength >= dictLimit)) { + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); From 2091f34e9e30f801e281b8000e26c9660786eb15 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 15:54:03 -0400 Subject: [PATCH 09/22] Find Proper Matches --- lib/compress/zstd_opt.c | 47 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 8d90ee7ff..bd740f854 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -664,9 +664,10 @@ U32 ZSTD_insertBtAndGetAllMatches ( matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; matches[mnum].len = (U32)matchLength; mnum++; - if (matchLength > ZSTD_OPT_NUM) break; - if (ip+matchLength == iLimit) { /* equal : no way to know if inf or sup */ - break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + if ( matchLength > ZSTD_OPT_NUM + | ip+matchLength == iLimit /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ } } @@ -687,6 +688,46 @@ U32 ZSTD_insertBtAndGetAllMatches ( *smallerPtr = *largerPtr = 0; + commonLengthSmaller = commonLengthLarger = 0; + + if (dictMode == ZSTD_dictMatchState && nbCompares) { + U32 dictMatchIndex = dms->hashTable[h]; + const U32* const dmsBt = dms->chainTable; + while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].len = (U32)matchLength; + mnum++; + if ( matchLength > ZSTD_OPT_NUM + | ip+matchLength == iLimit /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + } + assert(matchEndIdx > current+8); ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ return mnum; From 4bb79f9c559e7111d9a5c2c4a025565e8937904b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 16:59:49 -0400 Subject: [PATCH 10/22] Misc Changes --- lib/compress/zstd_opt.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index bd740f854..ea80d7805 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -612,7 +612,8 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const hashLog3 = ms->hashLog3; size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); U32 const dmsMatchIndex3 = dms->hashTable3[hash3]; - if (dmsMatchIndex3 >= dmsLowLimit) { + matchIndex3 = dmsMatchIndex3 + dmsIndexDelta; + if ((dmsMatchIndex3 >= dmsLowLimit) & (current - matchIndex3 < (1<<18))) { const BYTE* const match = dmsBase + dmsMatchIndex3; size_t mlen = ZSTD_count_2segments(ip, match, iLimit, dmsEnd, prefixStart); if (mlen >= mls) { @@ -664,8 +665,8 @@ U32 ZSTD_insertBtAndGetAllMatches ( matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; matches[mnum].len = (U32)matchLength; mnum++; - if ( matchLength > ZSTD_OPT_NUM - | ip+matchLength == iLimit /* equal : no way to know if inf or sup */) { + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ break; /* drop, to preserve bt consistency (miss a little bit of compression) */ } @@ -709,19 +710,18 @@ U32 ZSTD_insertBtAndGetAllMatches ( matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; matches[mnum].len = (U32)matchLength; mnum++; - if ( matchLength > ZSTD_OPT_NUM - | ip+matchLength == iLimit /* equal : no way to know if inf or sup */) { + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { break; /* drop, to guarantee consistency (miss a little bit of compression) */ } } + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ if (match[matchLength] < ip[matchLength]) { - if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ } else { /* match is larger than current */ - if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ commonLengthLarger = matchLength; dictMatchIndex = nextPtr[0]; } From 87fe4788a36019fcfbc0c1feb31341ab85b6bd72 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 14 Jun 2018 20:59:29 -0400 Subject: [PATCH 11/22] Fix Compression Ratio Regression #1 --- lib/compress/zstd_opt.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index ea80d7805..c22ddeb53 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -425,7 +425,7 @@ static U32 ZSTD_insertBt1( } #endif - if ((dictMode == ZSTD_noDict) || (matchIndex+matchLength >= dictLimit)) { + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); @@ -530,6 +530,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState || btMask >= dmsHighLimit ? 0 : dmsHighLimit - btMask; size_t bestLength = lengthToBeat-1; DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current); @@ -551,15 +552,14 @@ U32 ZSTD_insertBtAndGetAllMatches ( dmsBase + repIndex - dmsIndexDelta : dictBase + repIndex; assert(current >= windowLow); - if ( dictMode == ZSTD_extDict /* this case only valid in extDict mode */ + if ( dictMode == ZSTD_extDict && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; } if (dictMode == ZSTD_dictMatchState - && ( ((repOffset-1) /*intentional overflow*/ < current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ - & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; } } @@ -582,7 +582,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( if ((matchIndex3 > windowLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; - if ((dictMode == ZSTD_noDict) /*static*/ || (matchIndex3 >= dictLimit)) { + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { const BYTE* const match = base + matchIndex3; mlen = ZSTD_count(ip, match, iLimit); } else { @@ -716,7 +716,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( } } - if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ if (match[matchLength] < ip[matchLength]) { commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ From f0a13bcd68466f230ac5bc4dc92854779b5bc285 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 15 Jun 2018 14:08:58 -0400 Subject: [PATCH 12/22] Make Sure Position 0 Gets Into the Tree --- lib/compress/zstd_compress.c | 1 + lib/compress/zstd_opt.c | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 01ba01bda..216196ed6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -966,6 +966,7 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) ZSTD_window_clear(&ms->window); ms->nextToUpdate = ms->window.dictLimit + 1; + ms->nextToUpdate3 = ms->window.dictLimit + 1; ms->loadedDictEnd = 0; ms->opt.litLengthSum = 0; /* force reset of btopt stats */ ms->dictMatchState = NULL; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index c22ddeb53..92805b9b2 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -384,6 +384,7 @@ static U32 ZSTD_insertBt1( U32* largerPtr = smallerPtr + 1; U32 dummy32; /* to be nullified at the end */ U32 const windowLow = ms->window.lowLimit; + U32 const matchLow = windowLow ? windowLow : 1; U32 matchEndIdx = current+8+1; size_t bestLength = 8; U32 nbCompares = 1U << cParams->searchLog; @@ -399,7 +400,7 @@ static U32 ZSTD_insertBt1( assert(ip <= iend-8); /* required for h calculation */ hashTable[h] = current; /* Update Hash Table */ - while (nbCompares-- && (matchIndex > windowLow)) { + while (nbCompares-- && (matchIndex >= matchLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(matchIndex < current); @@ -517,6 +518,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* const prefixStart = base + dictLimit; U32 const btLow = btMask >= current ? 0 : current - btMask; U32 const windowLow = ms->window.lowLimit; + U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; U32 matchEndIdx = current+8+1; /* farthest referenced position of any match => detects repetitive patterns */ @@ -638,7 +640,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( hashTable[h] = current; /* Update Hash Table */ - while (nbCompares-- && (matchIndex > windowLow)) { + while (nbCompares-- && (matchIndex >= matchLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match; From de639502aa84bfa2ec46e1c8befdc583232332ed Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 19 Jun 2018 15:13:30 -0400 Subject: [PATCH 13/22] Update Dict Attachment Cut-Offs --- lib/compress/zstd_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 216196ed6..3ab0f9509 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1239,8 +1239,8 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, 32 KB, /* ZSTD_lazy */ 32 KB, /* ZSTD_lazy2 */ 32 KB, /* ZSTD_btlazy2 */ - 256 KB, /* ZSTD_btopt */ - 256 KB /* ZSTD_btultra */ + 32 KB, /* ZSTD_btopt */ + 8 KB /* ZSTD_btultra */ }; const int attachDict = ( pledgedSrcSize <= attachDictSizeCutoffs[cdict->cParams.strategy] || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN ) From 03c39c540b28a2f323d96105ec01bc5b000debc8 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 19 Jun 2018 15:36:33 -0400 Subject: [PATCH 14/22] Fix Incorrect Param --- lib/compress/zstd_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 92805b9b2..d494982a3 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1060,7 +1060,7 @@ size_t ZSTD_compressBlock_btopt( const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_btopt"); - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 0 /*optLevel*/, ZSTD_noDict); } From ae1f3898a20c612dd3c77eaa8fa48bc33ef9f695 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 19 Jun 2018 22:18:08 -0400 Subject: [PATCH 15/22] Remove Dead(!) HC3 DMS Lookup --- lib/compress/zstd_opt.c | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index d494982a3..5296d9603 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -580,7 +580,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); if ((matchIndex3 > windowLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; @@ -608,34 +608,8 @@ U32 ZSTD_insertBtAndGetAllMatches ( return 1; } } - } else if (dictMode == ZSTD_dictMatchState) { - /* should we perform this search even if the working ctx search - * found a match? */ - U32 const hashLog3 = ms->hashLog3; - size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); - U32 const dmsMatchIndex3 = dms->hashTable3[hash3]; - matchIndex3 = dmsMatchIndex3 + dmsIndexDelta; - if ((dmsMatchIndex3 >= dmsLowLimit) & (current - matchIndex3 < (1<<18))) { - const BYTE* const match = dmsBase + dmsMatchIndex3; - size_t mlen = ZSTD_count_2segments(ip, match, iLimit, dmsEnd, prefixStart); - if (mlen >= mls) { - matchIndex3 = dmsMatchIndex3 + dmsIndexDelta; - DEBUGLOG(8, "found small dms match with hlog3, of length %u", - (U32)mlen); - bestLength = mlen; - assert(current > matchIndex3); - assert(mnum==0); /* no prior solution */ - matches[0].off = (current - matchIndex3) + ZSTD_REP_MOVE; - matches[0].len = (U32)mlen; - mnum = 1; - if ( (mlen > sufficient_len) | - (ip+mlen == iLimit) ) { /* best possible length */ - ms->nextToUpdate = current+1; /* skip insertion */ - return 1; - } - } - } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ } hashTable[h] = current; /* Update Hash Table */ From 0a6cf7cd1d02b42fcdff17d1a538f021ad6c8345 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 20 Jun 2018 15:27:23 -0400 Subject: [PATCH 16/22] Minor Changes --- lib/compress/zstd_opt.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 5296d9603..0abdf3b90 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -359,7 +359,7 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* /** ZSTD_insertBt1() : add one or multiple positions to tree. * ip : assumed <= iend-8 . * @return : nb of positions added */ -static U32 ZSTD_insertBt1( +FORCE_INLINE_TEMPLATE U32 ZSTD_insertBt1( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, U32 const mls, const ZSTD_dictMode_e dictMode) @@ -665,11 +665,10 @@ U32 ZSTD_insertBtAndGetAllMatches ( *smallerPtr = *largerPtr = 0; - commonLengthSmaller = commonLengthLarger = 0; - if (dictMode == ZSTD_dictMatchState && nbCompares) { U32 dictMatchIndex = dms->hashTable[h]; const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ From 9c14eafe3d679472382253b6de4ac5e40baac69e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 20 Jun 2018 15:51:14 -0400 Subject: [PATCH 17/22] Also Use `matchLow` for HC3 Match --- lib/compress/zstd_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 0abdf3b90..90a262025 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -581,7 +581,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); - if ((matchIndex3 > windowLow) + if ((matchIndex3 >= matchLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { From 5d81f71e8390ddce11e186854d407f5d9bfc7805 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 20 Jun 2018 16:54:53 -0400 Subject: [PATCH 18/22] Consistency in Guarding DMS-Only Variable Initializations --- lib/compress/zstd_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 90a262025..3a6e4c45d 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -526,7 +526,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 mnum = 0; U32 nbCompares = 1U << cParams->searchLog; - const ZSTD_matchState_t* dms = ms->dictMatchState; + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; From 5da9bbc38ede591d6cf8464bfa28f1b7bba05e61 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 21 Jun 2018 11:20:01 -0400 Subject: [PATCH 19/22] Conceivably Dedup ZSTD_noDict and ZSTD_dictMatchState _insertBt1 Impls By reverting to the bool extDict flag, we call ZSTD_insertBt1 with the same const args in both non-extDict dictModes. --- lib/compress/zstd_opt.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 3a6e4c45d..1d0ca9ca4 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -359,10 +359,10 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* /** ZSTD_insertBt1() : add one or multiple positions to tree. * ip : assumed <= iend-8 . * @return : nb of positions added */ -FORCE_INLINE_TEMPLATE U32 ZSTD_insertBt1( +static U32 ZSTD_insertBt1( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iend, - U32 const mls, const ZSTD_dictMode_e dictMode) + U32 const mls, const int extDict) { U32* const hashTable = ms->hashTable; U32 const hashLog = cParams->hashLog; @@ -426,7 +426,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertBt1( } #endif - if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + if (!extDict || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); @@ -482,7 +482,7 @@ void ZSTD_updateTree_internal( idx, target, dictMode); while(idx < target) - idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, dictMode); + idx += ZSTD_insertBt1(ms, cParams, base+idx, iend, mls, dictMode == ZSTD_extDict); ms->nextToUpdate = target; } From 3caba150c6e962cf7f1efd5ad76b6bc6ef374781 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 21 Jun 2018 15:24:08 -0400 Subject: [PATCH 20/22] Fix `dmsBtLow` Test --- lib/compress/zstd_lazy.c | 2 +- lib/compress/zstd_opt.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 078d20360..bfe944928 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -166,7 +166,7 @@ static size_t ZSTD_DUBT_findBetterDictMatch ( U32* const dictBt = dms->chainTable; U32 const btLog = cParams->chainLog - 1; U32 const btMask = (1 << btLog) - 1; - U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? 0 : dictHighLimit - btMask; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; size_t commonLengthSmaller=0, commonLengthLarger=0; U32 matchEndIdx = current+8+1; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 1d0ca9ca4..3f7023f78 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -532,7 +532,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; - U32 const dmsBtLow = dictMode == ZSTD_dictMatchState || btMask >= dmsHighLimit ? 0 : dmsHighLimit - btMask; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && btMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - btMask : dmsLowLimit; size_t bestLength = lengthToBeat-1; DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current); From 5bd3d4b7d2dc660a25fe14e4d966ac947ac7b5c5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 21 Jun 2018 15:25:44 -0400 Subject: [PATCH 21/22] Add Debug Log Statement --- lib/compress/zstd_opt.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 3f7023f78..7bd7941d8 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -679,6 +679,8 @@ U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", + (U32)matchLength, current - matchIndex, current - matchIndex + ZSTD_REP_MOVE); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; From 3e91dc4d6a96b6122807c2e411851c5220212f91 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 21 Jun 2018 15:25:32 -0400 Subject: [PATCH 22/22] Add Repcode Bounds Check --- lib/compress/zstd_opt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 7bd7941d8..476cdc148 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -561,7 +561,8 @@ U32 ZSTD_insertBtAndGetAllMatches ( repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; } if (dictMode == ZSTD_dictMatchState - && ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && ( ((repOffset-1) /*intentional overflow*/ < current - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `current > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; } }