From a85ecb32bdae683a6d2378253fc5c4bd62147211 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 16 May 2018 02:26:07 -0400 Subject: [PATCH 01/26] Add dictMode Param to ZSTD_compressBlock_lazy_generic --- lib/compress/zstd_lazy.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 9f158123f..96aeeed5b 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -464,7 +464,8 @@ size_t ZSTD_compressBlock_lazy_generic( U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth) + const U32 searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) { const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; @@ -479,6 +480,8 @@ size_t ZSTD_compressBlock_lazy_generic( searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + (void)dictMode; + /* init */ ip += (ip==base); ms->nextToUpdate3 = ms->nextToUpdate; @@ -597,28 +600,28 @@ size_t ZSTD_compressBlock_btlazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2, ZSTD_noDict); } size_t ZSTD_compressBlock_lazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2, ZSTD_noDict); } size_t ZSTD_compressBlock_lazy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1, ZSTD_noDict); } size_t ZSTD_compressBlock_greedy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0, ZSTD_noDict); } From 90cfc799e5728d43a6a2aea554c7110714b39baf Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 16 May 2018 02:30:20 -0400 Subject: [PATCH 02/26] Add _dictMatchState Stubs for ZSTD_lazy Functions --- lib/compress/zstd_compress.c | 28 +++++++++++++++++++++------- lib/compress/zstd_lazy.c | 28 ++++++++++++++++++++++++++++ lib/compress/zstd_lazy.h | 13 +++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d79e9c74b..b075d1123 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2230,17 +2230,31 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo { static const ZSTD_blockCompressor blockCompressor[3][(unsigned)ZSTD_btultra+1] = { { ZSTD_compressBlock_fast /* default for 0 */, - ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, - ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, - ZSTD_compressBlock_btopt, ZSTD_compressBlock_btultra }, + ZSTD_compressBlock_fast, + ZSTD_compressBlock_doubleFast, + ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, + ZSTD_compressBlock_lazy2, + ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, + ZSTD_compressBlock_btultra }, { ZSTD_compressBlock_fast_extDict /* default for 0 */, - ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, - ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, - ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict }, + ZSTD_compressBlock_fast_extDict, + ZSTD_compressBlock_doubleFast_extDict, + ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict, + ZSTD_compressBlock_lazy2_extDict, + ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict }, { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, ZSTD_compressBlock_fast_dictMatchState, ZSTD_compressBlock_doubleFast_dictMatchState, - NULL, NULL, NULL, NULL, NULL, NULL /* unimplemented as of yet */ } + ZSTD_compressBlock_greedy_dictMatchState, + ZSTD_compressBlock_lazy_dictMatchState, + ZSTD_compressBlock_lazy2_dictMatchState, + ZSTD_compressBlock_btlazy2_dictMatchState, + NULL, NULL /* unimplemented as of yet */ } }; ZSTD_blockCompressor selectedCompressor; ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 96aeeed5b..2f74393b8 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -624,6 +624,34 @@ size_t ZSTD_compressBlock_greedy( return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0, ZSTD_noDict); } +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 1, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, cParams, src, srcSize, 0, 0, ZSTD_dictMatchState); +} + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_lazy_extDict_generic( diff --git a/lib/compress/zstd_lazy.h b/lib/compress/zstd_lazy.h index bda064f19..c299de6dc 100644 --- a/lib/compress/zstd_lazy.h +++ b/lib/compress/zstd_lazy.h @@ -36,6 +36,19 @@ size_t ZSTD_compressBlock_greedy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); + size_t ZSTD_compressBlock_greedy_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); From 33e2240fac7b993ac801ffaf924ec25e44904305 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 16 May 2018 02:31:30 -0400 Subject: [PATCH 03/26] Attach Dict When Using ZSTD_lazy Strategies --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b075d1123..142ff6c59 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1244,7 +1244,7 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN ) && !params.forceWindow /* dictMatchState isn't correctly * handled in _enforceMaxDist */ - && cdict->cParams.strategy <= ZSTD_dfast + && cdict->cParams.strategy <= ZSTD_btlazy2 && ZSTD_equivalentCParams(cctx->appliedParams.cParams, cdict->cParams); From 529d3a5acd9551bcb8abe87d16aa896326d2c32d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 16 May 2018 04:07:09 -0400 Subject: [PATCH 04/26] Convert Existing U32 extDict Vars to ZSTD_dictMode Enums --- lib/compress/zstd_lazy.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 2f74393b8..26101b954 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -62,7 +62,7 @@ void ZSTD_updateDUBT( static void ZSTD_insertDUBT1( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, U32 current, const BYTE* inputEnd, - U32 nbCompares, U32 btLow, int extDict) + U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode) { U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; @@ -92,10 +92,12 @@ static void ZSTD_insertDUBT1( size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(matchIndex < current); - if ( (!extDict) + if ( (dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ || (current < dictLimit) /* both in extDict */) { - const BYTE* const mBase = !extDict || ((matchIndex+matchLength) >= dictLimit) ? base : dictBase; + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ || (current < dictLimit) ); match = mBase + matchIndex; @@ -143,7 +145,7 @@ static size_t ZSTD_DUBT_findBestMatch ( const BYTE* const ip, const BYTE* const iend, size_t* offsetPtr, U32 const mls, - U32 const extDict) + const ZSTD_dictMode_e dictMode) { U32* const hashTable = ms->hashTable; U32 const hashLog = cParams->hashLog; @@ -196,7 +198,7 @@ static size_t ZSTD_DUBT_findBestMatch ( U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; U32 const nextCandidateIdx = *nextCandidateIdxPtr; ZSTD_insertDUBT1(ms, cParams, matchIndex, iend, - nbCandidates, unsortLimit, extDict); + nbCandidates, unsortLimit, dictMode); matchIndex = nextCandidateIdx; nbCandidates++; } @@ -221,7 +223,7 @@ static size_t ZSTD_DUBT_findBestMatch ( size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match; - if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { match = base + matchIndex; matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); } else { @@ -281,7 +283,7 @@ static size_t ZSTD_BtFindBestMatch ( DEBUGLOG(7, "ZSTD_BtFindBestMatch"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, 0); + return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, ZSTD_noDict); } @@ -311,7 +313,7 @@ static size_t ZSTD_BtFindBestMatch_extDict ( DEBUGLOG(7, "ZSTD_BtFindBestMatch_extDict"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, 1); + return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, ZSTD_extDict); } @@ -376,7 +378,7 @@ size_t ZSTD_HcFindBestMatch_generic ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 mls, const U32 extDict) + const U32 mls, const ZSTD_dictMode_e dictMode) { U32* const chainTable = ms->chainTable; const U32 chainSize = (1 << cParams->chainLog); @@ -397,7 +399,7 @@ size_t ZSTD_HcFindBestMatch_generic ( for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { size_t currentMl=0; - if ((!extDict) || matchIndex >= dictLimit) { + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; if (match[ml] == ip[ml]) /* potentially better */ currentMl = ZSTD_count(ip, match, iLimit); @@ -431,10 +433,10 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, 0); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, 0); + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_noDict); case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, 0); + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_noDict); } } @@ -447,10 +449,10 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, 1); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, 1); + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_extDict); case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, 1); + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_extDict); } } From f84fc63a4362c2df09e8faf72fbf8fece16638da Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 16 May 2018 04:32:21 -0400 Subject: [PATCH 05/26] Further Templatize Intermediate Functions on dictMode --- lib/compress/zstd_lazy.c | 85 ++++++++++------------------------------ 1 file changed, 21 insertions(+), 64 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 26101b954..19f6fe0aa 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -278,57 +278,29 @@ static size_t ZSTD_BtFindBestMatch ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 mls /* template */) + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) { DEBUGLOG(7, "ZSTD_BtFindBestMatch"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, ZSTD_noDict); + return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, dictMode); } static size_t ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(cParams->searchLength) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4); - case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6); - } -} - - -/** Tree updater, providing best match */ -static size_t ZSTD_BtFindBestMatch_extDict ( - ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, - const U32 mls) -{ - DEBUGLOG(7, "ZSTD_BtFindBestMatch_extDict"); - if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateDUBT(ms, cParams, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, cParams, ip, iLimit, offsetPtr, mls, ZSTD_extDict); -} - - -static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( - ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) + const ZSTD_dictMode_e dictMode) { switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 4); - case 5 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 5); + case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, dictMode); + case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, dictMode); case 7 : - case 6 : return ZSTD_BtFindBestMatch_extDict(ms, cParams, ip, iLimit, offsetPtr, 6); + case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, dictMode); } } @@ -428,31 +400,15 @@ size_t ZSTD_HcFindBestMatch_generic ( FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) + size_t* offsetPtr, const ZSTD_dictMode_e dictMode) { switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_noDict); + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, dictMode); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, dictMode); case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_noDict); - } -} - - -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* const iLimit, - size_t* const offsetPtr) -{ - switch(cParams->searchLength) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_extDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_extDict); + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, dictMode); } } @@ -478,7 +434,7 @@ size_t ZSTD_compressBlock_lazy_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, const ZSTD_dictMode_e dictMode); searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; @@ -507,7 +463,7 @@ size_t ZSTD_compressBlock_lazy_generic( /* first search (depth 0) */ { size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound, dictMode); if (ml2 > matchLength) matchLength = ml2, start = ip, offset=offsetFound; } @@ -529,7 +485,7 @@ size_t ZSTD_compressBlock_lazy_generic( matchLength = mlRep, offset = 0, start = ip; } { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -548,7 +504,7 @@ size_t ZSTD_compressBlock_lazy_generic( matchLength = ml2, offset = 0, start = ip; } { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -663,6 +619,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const void* src, size_t srcSize, const U32 searchMethod, const U32 depth) { + const ZSTD_dictMode_e dictMode = ZSTD_extDict; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; @@ -678,8 +635,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, const ZSTD_dictMode_e dictMode); + searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; U32 offset_1 = rep[0], offset_2 = rep[1]; @@ -708,7 +665,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* first search (depth 0) */ { size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound, dictMode); if (ml2 > matchLength) matchLength = ml2, start = ip, offset=offsetFound; } @@ -741,7 +698,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* search match, depth 1 */ { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -771,7 +728,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* search match, depth 2 */ { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { From 5d90708a0ad3e5922c732f60ac2213abf74238f9 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 1 Jun 2018 14:23:14 -0400 Subject: [PATCH 06/26] Go Back to Separate Intermediate Functions for Different Dict Modes --- lib/compress/zstd_lazy.c | 104 +++++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 20 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 19f6fe0aa..67289a0a4 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -291,16 +291,47 @@ static size_t ZSTD_BtFindBestMatch ( static size_t ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, - const ZSTD_dictMode_e dictMode) + size_t* offsetPtr) { switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, dictMode); - case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, dictMode); + case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_noDict); case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, dictMode); + case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + } +} + + +static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(cParams->searchLength) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); + case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); + case 7 : + case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + } +} + + +static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(cParams->searchLength) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + case 7 : + case 6 : return ZSTD_BtFindBestMatch(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_extDict); } } @@ -400,15 +431,47 @@ size_t ZSTD_HcFindBestMatch_generic ( FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr, const ZSTD_dictMode_e dictMode) + size_t* offsetPtr) { switch(cParams->searchLength) { default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, dictMode); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, dictMode); + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_noDict); case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, dictMode); + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + } +} + + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(cParams->searchLength) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + } +} + + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( + ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr) +{ + switch(cParams->searchLength) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(ms, cParams, ip, iLimit, offsetPtr, 6, ZSTD_extDict); } } @@ -434,8 +497,10 @@ size_t ZSTD_compressBlock_lazy_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, const ZSTD_dictMode_e dictMode); - searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + searchMax_f const searchMax = dictMode == ZSTD_dictMatchState ? + (searchMethod ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) : + (searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS); U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; (void)dictMode; @@ -463,7 +528,7 @@ size_t ZSTD_compressBlock_lazy_generic( /* first search (depth 0) */ { size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound, dictMode); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); if (ml2 > matchLength) matchLength = ml2, start = ip, offset=offsetFound; } @@ -485,7 +550,7 @@ size_t ZSTD_compressBlock_lazy_generic( matchLength = mlRep, offset = 0, start = ip; } { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -504,7 +569,7 @@ size_t ZSTD_compressBlock_lazy_generic( matchLength = ml2, offset = 0, start = ip; } { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -619,7 +684,6 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const void* src, size_t srcSize, const U32 searchMethod, const U32 depth) { - const ZSTD_dictMode_e dictMode = ZSTD_extDict; const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; @@ -635,8 +699,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, const ZSTD_dictMode_e dictMode); - searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; U32 offset_1 = rep[0], offset_2 = rep[1]; @@ -665,7 +729,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* first search (depth 0) */ { size_t offsetFound = 99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound, dictMode); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); if (ml2 > matchLength) matchLength = ml2, start = ip, offset=offsetFound; } @@ -698,7 +762,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* search match, depth 1 */ { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { @@ -728,7 +792,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* search match, depth 2 */ { size_t offset2=99999999; - size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2, dictMode); + size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { From c14db9445013c4d3979c155fa7de0227e4ef3b68 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 23 May 2018 15:10:18 -0400 Subject: [PATCH 07/26] Rename `base` -> `prefixLowest` --- lib/compress/zstd_lazy.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 67289a0a4..d9695f10a 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -493,7 +493,7 @@ size_t ZSTD_compressBlock_lazy_generic( const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const BYTE* const base = ms->window.base + ms->window.dictLimit; + const BYTE* const prefixLowest = ms->window.base + ms->window.dictLimit; typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, @@ -506,9 +506,9 @@ size_t ZSTD_compressBlock_lazy_generic( (void)dictMode; /* init */ - ip += (ip==base); + ip += (ip==prefixLowest); ms->nextToUpdate3 = ms->nextToUpdate; - { U32 const maxRep = (U32)(ip-base); + { U32 const maxRep = (U32)(ip-prefixLowest); if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; } @@ -586,7 +586,7 @@ size_t ZSTD_compressBlock_lazy_generic( */ /* catch up */ if (offset) { - while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > base)) + while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > prefixLowest)) && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ { start--; matchLength++; } offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); From f74c2cd6731aa124cfa503528c3dcb1aedac7794 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 23 May 2018 14:13:16 -0400 Subject: [PATCH 08/26] Disallow Too-Long Repcodes When Using an Attached Dict --- lib/compress/zstd_lazy.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index d9695f10a..4d4d81e1f 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -503,15 +503,33 @@ size_t ZSTD_compressBlock_lazy_generic( (searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS); U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = dictMode == ZSTD_dictMatchState ? + dms->window.dictLimit : 0; + const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? + dms->window.base : NULL; + const BYTE* const dictLowest = dictMode == ZSTD_dictMatchState ? + dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? + dms->window.nextSrc : NULL; + const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest); + (void)dictMode; /* init */ - ip += (ip==prefixLowest); + ip += (dictAndPrefixLength == 0); ms->nextToUpdate3 = ms->nextToUpdate; - { U32 const maxRep = (U32)(ip-prefixLowest); + if (dictMode == ZSTD_noDict) { + U32 const maxRep = (U32)(ip - prefixLowest); if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; } + if (dictMode == ZSTD_dictMatchState) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } /* Match Loop */ while (ip < ilimit) { From a2a24bebecf7a1ac14b503129d08f1b4f6c9b40b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 23 May 2018 15:06:24 -0400 Subject: [PATCH 09/26] First Repcode Check --- lib/compress/zstd_lazy.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 4d4d81e1f..14fd275a7 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -493,7 +493,9 @@ size_t ZSTD_compressBlock_lazy_generic( const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const BYTE* const prefixLowest = ms->window.base + ms->window.dictLimit; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, @@ -512,6 +514,9 @@ size_t ZSTD_compressBlock_lazy_generic( dictBase + dictLowestIndex : NULL; const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest); (void)dictMode; @@ -537,6 +542,12 @@ size_t ZSTD_compressBlock_lazy_generic( size_t offset=0; const BYTE* start=ip+1; + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = (dictMode == ZSTD_dictMatchState + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + /* check repCode */ if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) { /* repcode : we take it */ @@ -544,6 +555,18 @@ size_t ZSTD_compressBlock_lazy_generic( if (depth==0) goto _storeSequence; } + if (dictMode == ZSTD_dictMatchState + && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, istart) + 4; + if (depth==0) goto _storeSequence; + } else if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + /* first search (depth 0) */ { size_t offsetFound = 99999999; size_t const ml2 = searchMax(ms, cParams, ip, iend, &offsetFound); From 3b82a23a3540028d1be15c0ef2769ac93b0cfbdd Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 23 May 2018 15:49:43 -0400 Subject: [PATCH 10/26] Second Repcode Check --- lib/compress/zstd_lazy.c | 43 ++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 14fd275a7..e90f51a74 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -640,16 +640,39 @@ _storeSequence: } /* check immediate repcode */ - while ( ((ip <= ilimit) & (offset_2>0)) - && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { - /* store sequence */ - matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } } + if (dictMode == ZSTD_dictMatchState) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState + && repIndex2 < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, istart) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = offset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } /* Save reps for next block */ rep[0] = offset_1 ? offset_1 : savedOffset; From 2e93736a7786dfb73cf32a2e75ae5679c375ba60 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 29 May 2018 16:05:48 -0400 Subject: [PATCH 11/26] Remove Pre-Existing Repcode Check --- lib/compress/zstd_lazy.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index e90f51a74..3a82ef48d 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -549,12 +549,6 @@ size_t ZSTD_compressBlock_lazy_generic( base + repIndex; /* check repCode */ - if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) { - /* repcode : we take it */ - matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; - if (depth==0) goto _storeSequence; - } - if (dictMode == ZSTD_dictMatchState && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { From 211a61b69b919d0963ed0ce9de943a20866bef1b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 29 May 2018 16:06:23 -0400 Subject: [PATCH 12/26] Focus on Non-BT Impls for the Moment --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 142ff6c59..f42c4595d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1244,7 +1244,7 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN ) && !params.forceWindow /* dictMatchState isn't correctly * handled in _enforceMaxDist */ - && cdict->cParams.strategy <= ZSTD_btlazy2 + && cdict->cParams.strategy <= ZSTD_lazy2 && ZSTD_equivalentCParams(cctx->appliedParams.cParams, cdict->cParams); From 6204b6d592417e368d15ee08ef85a658234b85f7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 29 May 2018 16:26:23 -0400 Subject: [PATCH 13/26] Check Dict Match State in ZSTD_HcFindBestMatch_generic --- lib/compress/zstd_lazy.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 3a82ef48d..334d5dbc7 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -424,6 +424,37 @@ size_t ZSTD_HcFindBestMatch_generic ( matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); } + if (dictMode == ZSTD_dictMatchState) { + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > chainSize ? dmsSize - chainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, cParams->hashLog, mls)]; + + for ( ; (matchIndex>dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = current - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) break; + matchIndex = dmsChainTable[matchIndex & chainMask]; + } + } + return ml; } From c87dd2121d4963dc48edc7c846b1f442ee3c8147 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 31 May 2018 18:32:41 -0400 Subject: [PATCH 14/26] Update Dictionary Attachment Cutoff Values --- lib/compress/zstd_compress.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f42c4595d..79d0741ee 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1233,12 +1233,12 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, 8 KB, /* unused */ 8 KB, /* ZSTD_fast */ 16 KB, /* ZSTD_dfast */ - 16 KB, /* ZSTD_greedy */ - 16 KB, /* ZSTD_lazy */ - 16 KB, /* ZSTD_lazy2 */ - 16 KB, /* ZSTD_btlazy2 */ - 16 KB, /* ZSTD_btopt */ - 16 KB /* ZSTD_btultra */ + 128 KB, /* ZSTD_greedy */ + 128 KB, /* ZSTD_lazy */ + 128 KB, /* ZSTD_lazy2 */ + 128 KB, /* ZSTD_btlazy2 */ + 128 KB, /* ZSTD_btopt */ + 128 KB /* ZSTD_btultra */ }; const int attachDict = ( pledgedSrcSize <= attachDictSizeCutoffs[cdict->cParams.strategy] || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN ) From 555ab9f8cf2353a56ec5d23b133c44ec33e9e27e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 6 Jun 2018 17:34:29 -0400 Subject: [PATCH 15/26] Apply Match Continuation Bug Fix --- lib/compress/zstd_lazy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 334d5dbc7..e9f5e1d35 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -584,7 +584,7 @@ size_t ZSTD_compressBlock_lazy_generic( && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, istart) + 4; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; if (depth==0) goto _storeSequence; } else if ( dictMode == ZSTD_noDict && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { @@ -676,7 +676,7 @@ _storeSequence: if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, istart) + 4; + matchLength = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = offset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; From 338bede9b51b2571e6718475d4ba6392c1452da6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 6 Jun 2018 19:54:13 -0400 Subject: [PATCH 16/26] Also Implement Depth Repcode Checks --- lib/compress/zstd_lazy.c | 91 +++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index e9f5e1d35..77b0b8bcf 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -573,21 +573,22 @@ size_t ZSTD_compressBlock_lazy_generic( size_t offset=0; const BYTE* start=ip+1; - const U32 repIndex = (U32)(ip - base) + 1 - offset_1; - const BYTE* repMatch = (dictMode == ZSTD_dictMatchState - && repIndex < prefixLowestIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - /* check repCode */ - if (dictMode == ZSTD_dictMatchState - && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; - if (depth==0) goto _storeSequence; - } else if ( dictMode == ZSTD_noDict - && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + if (dictMode == ZSTD_dictMatchState) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = (dictMode == ZSTD_dictMatchState + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } + } + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; if (depth==0) goto _storeSequence; } @@ -608,13 +609,29 @@ size_t ZSTD_compressBlock_lazy_generic( if (depth>=1) while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + if ( (dictMode == ZSTD_noDict) + && (offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(mlRep * 3); int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); if ((mlRep >= 4) && (gain2 > gain1)) matchLength = mlRep, offset = 0, start = ip; } + if (dictMode == ZSTD_dictMatchState) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + } { size_t offset2=99999999; size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ @@ -627,12 +644,28 @@ size_t ZSTD_compressBlock_lazy_generic( /* let's find an even better one */ if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { - size_t const ml2 = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; - int const gain2 = (int)(ml2 * 4); + if ( (dictMode == ZSTD_noDict) + && (offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); - if ((ml2 >= 4) && (gain2 > gain1)) - matchLength = ml2, offset = 0, start = ip; + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + if (dictMode == ZSTD_dictMatchState) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } } { size_t offset2=99999999; size_t const ml2 = searchMax(ms, cParams, ip, iend, &offset2); @@ -668,15 +701,15 @@ _storeSequence: if (dictMode == ZSTD_dictMatchState) { while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState - && repIndex2 < prefixLowestIndex ? - dictBase - dictIndexDelta + repIndex2 : - base + repIndex2; - if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = dictMode == ZSTD_dictMatchState + && repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = offset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; From 2162aa9f1857d23aea5feffb75001e6b2a734017 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 7 Jun 2018 16:55:13 -0400 Subject: [PATCH 17/26] Do Not Inline DMS Search Function --- lib/compress/zstd_lazy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 77b0b8bcf..ba552d252 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -475,7 +475,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( } -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( +static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) From d53a04211c28b6c73f2a0005ec7e8ee5c454cef7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 7 Jun 2018 16:57:45 -0400 Subject: [PATCH 18/26] Update Dictionary Attachment Cutoff Values Again --- lib/compress/zstd_compress.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 79d0741ee..807b1df5b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1233,12 +1233,12 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, 8 KB, /* unused */ 8 KB, /* ZSTD_fast */ 16 KB, /* ZSTD_dfast */ - 128 KB, /* ZSTD_greedy */ - 128 KB, /* ZSTD_lazy */ - 128 KB, /* ZSTD_lazy2 */ - 128 KB, /* ZSTD_btlazy2 */ - 128 KB, /* ZSTD_btopt */ - 128 KB /* ZSTD_btultra */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 256 KB, /* ZSTD_btlazy2 */ + 256 KB, /* ZSTD_btopt */ + 256 KB /* ZSTD_btultra */ }; const int attachDict = ( pledgedSrcSize <= attachDictSizeCutoffs[cdict->cParams.strategy] || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN ) From b82063b2667cba0d7b025b3b6a31cfda50bdae7c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 8 Jun 2018 15:06:47 -0400 Subject: [PATCH 19/26] Extend Dictionary Matches Backwards --- lib/compress/zstd_lazy.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index ba552d252..a63656dad 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -685,9 +685,17 @@ size_t ZSTD_compressBlock_lazy_generic( */ /* catch up */ if (offset) { - while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > prefixLowest)) - && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ - { start--; matchLength++; } + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > prefixLowest)) + && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (dictMode == ZSTD_dictMatchState) { + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); } /* store sequence */ From d53200a846d831c5bbcc9e3933356c235d5e45c3 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Sat, 9 Jun 2018 17:19:04 -0400 Subject: [PATCH 20/26] Fix Cast Warning --- lib/compress/zstd_lazy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index a63656dad..ca86c3e63 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -718,7 +718,7 @@ _storeSequence: && (MEM_read32(repMatch) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = offset; /* swap offset_2 <=> offset_1 */ + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; From fc682263d0bcc44715c85e936fbef4bb8cdda9ab Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jun 2018 20:02:33 -0400 Subject: [PATCH 21/26] fixed g_debuglevel variable name in debug.h --- lib/common/debug.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/common/debug.h b/lib/common/debug.h index 1b81f229c..95a3fcabe 100644 --- a/lib/common/debug.h +++ b/lib/common/debug.h @@ -93,18 +93,18 @@ extern "C" { #if (DEBUGLEVEL>=2) # include -extern int g_debug_level; /* here, this variable is only declared, +extern int g_debuglevel; /* here, this variable is only declared, it actually lives in debug.c, and is shared by the whole process. It's typically used to enable very verbose levels on selective conditions (such as position in src) */ # define RAWLOG(l, ...) { \ - if (l<=g_debug_level) { \ + if (l<=g_debuglevel) { \ fprintf(stderr, __VA_ARGS__); \ } } # define DEBUGLOG(l, ...) { \ - if (l<=g_debug_level) { \ + if (l<=g_debuglevel) { \ fprintf(stderr, __FILE__ ": " __VA_ARGS__); \ fprintf(stderr, " \n"); \ } } From 7fee966f02ddd037f001096158281273a61c442d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jun 2018 10:22:24 -0400 Subject: [PATCH 22/26] fix dctx initialization within ZSTD_decompress in stack mode when ZSTD_HEAPMODE=0 (which is not default). Also : added an associated test (test-fuzzer-stackmode) run on travis CI fix #1186 --- .travis.yml | 5 +++++ lib/decompress/zstd_decompress.c | 5 +++-- tests/Makefile | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b01805bee..8c5eb70f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,14 +20,18 @@ matrix: - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' + - env: Cmd='make -C tests test-fuzzer-stackmode' + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' - env: Cmd='make arminstall && make armfuzz' + # Following test is disabled, as there is a bug in Travis' ld # preventing aarch64 compilation to complete. # > collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped # to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug # - env: Cmd='make arminstall && make aarch64fuzz' + - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' @@ -35,6 +39,7 @@ matrix: - env: Cmd='make lz4install && make -C tests test-lz4' - env: Cmd='bash tests/libzstd_partial_builds.sh' + # tag-specific test - if: tag =~ ^v[0-9]\.[0-9] env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH' diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 030b74839..f65d85662 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -192,6 +192,8 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->inBuffSize = 0; dctx->outBuffSize = 0; dctx->streamStage = zdss_init; + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); } @@ -215,8 +217,6 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem); if (!dctx) return NULL; dctx->customMem = customMem; - dctx->legacyContext = NULL; - dctx->previousLegacyVersion = 0; ZSTD_initDCtx_internal(dctx); return dctx; } @@ -1996,6 +1996,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr return regenSize; #else /* stack mode */ ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); #endif } diff --git a/tests/Makefile b/tests/Makefile index c1482bc9c..792ee9efe 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -360,6 +360,9 @@ test-fullbench32: fullbench32 datagen test-fuzzer: fuzzer $(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS) +test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0 +test-fuzzer-stackmode: test-fuzzer + test-fuzzer32: fuzzer32 $(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS) From bf30b9caf4d8afa907fd1b3424475fccbd48a915 Mon Sep 17 00:00:00 2001 From: Ryan Schmidt Date: Thu, 14 Jun 2018 15:05:33 -0500 Subject: [PATCH 23/26] Add CXXFLAGS to ALL_LDFLAGS (#1178) pzstd requires C++11, which older C++ standard libraries like libstdc++ as used on OS X 10.8 and earlier don't support. The user might address this by setting "CXXFLAGS=-stdlib=libc++". This flag must be used both at compile time and at link time. Asking the user to also put the flag in LDFLAGS is undesirable because then the flag would also be used when linking C code, which would be inappropriate. --- contrib/pzstd/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index e15795f72..14b932297 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -42,7 +42,7 @@ PZSTD_LDFLAGS = EXTRA_FLAGS = ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS) ALL_CXXFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS) -ALL_LDFLAGS = $(EXTRA_FLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS) +ALL_LDFLAGS = $(EXTRA_FLAGS) $(CXXFLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS) # gtest libraries need to go before "-lpthread" because they depend on it. From 78e5e2887ec6ee98b557070092ab46d160a7e875 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jun 2018 15:51:31 -0400 Subject: [PATCH 24/26] Visual Studio project blind fix --- build/VS2008/fullbench/fullbench.vcproj | 4 ++++ build/VS2008/fuzzer/fuzzer.vcproj | 4 ++++ build/VS2008/zstd/zstd.vcproj | 4 ++++ build/VS2008/zstdlib/zstdlib.vcproj | 4 ++++ build/VS2010/fullbench/fullbench.vcxproj | 1 + build/VS2010/fuzzer/fuzzer.vcxproj | 1 + build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 1 + build/VS2010/libzstd/libzstd.vcxproj | 1 + build/VS2010/zstd/zstd.vcxproj | 1 + 9 files changed, 21 insertions(+) diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index 715ea2579..2ce7f74e7 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -332,6 +332,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index 1421619a1..f6ea1f855 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -352,6 +352,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index dbd211c06..5d9f68327 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -356,6 +356,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index 340a4cd89..7234b02ac 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -348,6 +348,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index d0cbcae98..2ec8c4f61 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -156,6 +156,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 6fe327209..ac400028b 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -158,6 +158,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 2d04c6935..db9fdedbb 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -22,6 +22,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index c01a5d179..9550b03cf 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -22,6 +22,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index ace343465..36a265a33 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -20,6 +20,7 @@ + From 6901c94cd67c9a6ca0a3f2bc78abf6e3e210d216 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jun 2018 19:47:05 -0400 Subject: [PATCH 25/26] avoid duplicate code comments when a function is decribed in hist.h, do not describe it again in hist.c to avoid future doc synchronization issues. --- lib/compress/hist.c | 13 ++++--------- lib/compress/hist.h | 8 +++++--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/compress/hist.c b/lib/compress/hist.c index da3b749db..16524756b 100644 --- a/lib/compress/hist.c +++ b/lib/compress/hist.c @@ -46,14 +46,6 @@ unsigned HIST_isError(size_t code) { return ERR_isError(code); } /*-************************************************************** * Histogram functions ****************************************************************/ -/*! HIST_count_simple : - Counts byte values within `src`, storing histogram into table `count`. - Doesn't use any additional memory, very limited stack usage. - But unsafe : doesn't check that all values within `src` fit into `count`. - For this reason, prefer using a table `count` of size 256. - @return : count of most numerous element. - this function doesn't produce any error (i.e. it must succeed). -*/ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) { @@ -83,7 +75,10 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, /* HIST_count_parallel_wksp() : - * Same as HIST_count_parallel(), but using an externally provided scratch buffer. + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. * `workSpace` size must be a table of size >= HIST_WKSP_SIZE_U32. * @return : largest histogram frequency, * or an error code (notably when histogram would be larger than *maxSymbolValuePtr). */ diff --git a/lib/compress/hist.h b/lib/compress/hist.h index 293b188bc..788470da7 100644 --- a/lib/compress/hist.h +++ b/lib/compress/hist.h @@ -81,10 +81,12 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, unsigned* workSpace); /*! HIST_count_simple() : - * Same as HIST_countFast(), but does not use any additional memory (not even on stack). - * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. * It is also a bit slower for large inputs. - * This function doesn't produce any error (i.e. it must succeed). + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). */ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); From 507bef196deb994676f06d1c498dcdfd859cc168 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jun 2018 19:48:40 -0400 Subject: [PATCH 26/26] added debug.h for cmake --- build/cmake/lib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 2cdfde0f3..ade946daf 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -55,6 +55,7 @@ SET(Sources SET(Headers ${LIBRARY_DIR}/zstd.h + ${LIBRARY_DIR}/common/debug.h ${LIBRARY_DIR}/common/pool.h ${LIBRARY_DIR}/common/threading.h ${LIBRARY_DIR}/common/bitstream.h