From 89c9e1a847625aa143b79735a8f5a4633e95925a Mon Sep 17 00:00:00 2001 From: inikep Date: Sun, 6 Mar 2016 23:21:52 +0100 Subject: [PATCH 1/3] added missing "case 3:" in ZSTD_BtGetAllMatches --- lib/zstd_internal.h | 2 +- lib/zstd_opt.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index b06097a21..462161bd7 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -105,7 +105,7 @@ static const size_t ZSTD_frameHeaderSize_min = 5; #define OffFSELog 9 #define MaxSeq MAX(MaxLL, MaxML) -#define LONGNBSEQ 0xFF00 +#define LONGNBSEQ 0x7F00 #define FSE_ENCODING_RAW 0 #define FSE_ENCODING_RLE 1 diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 780c5d415..c6205089f 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -342,6 +342,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS ( switch(matchLengthSearch) { default : + case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches); case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches); case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches); case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches); @@ -369,6 +370,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( switch(matchLengthSearch) { default : + case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches); case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches); case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches); case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches); From e2446b0e36d01e4452ac23f6913b361242dac466 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 7 Mar 2016 10:07:08 +0100 Subject: [PATCH 2/3] added support for MINMATCH=3 on big endian architecture changed ZSTD_HASHLOG3_MIN to 2 (4 bytes for MINMACH > 3) --- lib/mem.h | 25 ++++++++++++++----------- lib/zstd_compress.c | 2 +- lib/zstd_opt.h | 14 ++++++-------- lib/zstd_static.h | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/mem.h b/lib/mem.h index a369033b1..ceafd57b4 100644 --- a/lib/mem.h +++ b/lib/mem.h @@ -187,17 +187,6 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ - -MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : return MEM_read32(memPtr)<<8; - } -} - MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -286,6 +275,20 @@ MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) MEM_writeLE64(memPtr, (U64)val); } + /* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + #if defined (__cplusplus) } #endif diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index e24016c7a..87e8e72ee 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -876,7 +876,7 @@ static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE ***************************************/ static const U32 prime3bytes = 506832829U; static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } -static size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_read32(ptr), h); } +static size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } static const U32 prime4bytes = 2654435761U; static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index c6205089f..0c4a80f77 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -335,10 +335,9 @@ static U32 ZSTD_BtGetAllMatches ( static U32 ZSTD_BtGetAllMatches_selectMLS ( ZSTD_CCtx* zc, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, + const BYTE* ip, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches) { - (void)iLowLimit; /* unused */ switch(matchLengthSearch) { default : @@ -363,10 +362,9 @@ static U32 ZSTD_BtGetAllMatches_extDict ( static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( ZSTD_CCtx* zc, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, + const BYTE* ip, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches) { - (void)iLowLimit; switch(matchLengthSearch) { default : @@ -447,7 +445,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -567,7 +565,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } while (mlen >= minMatch); } - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, ip, iend, maxSearches, mls, matches); + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && matches[match_num-1].len > sufficient_len) { @@ -779,7 +777,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, best_mlen = (last_pos) ? last_pos : minMatch; - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */ ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -909,7 +907,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, ip, iend, maxSearches, mls, matches); + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && matches[match_num-1].len > sufficient_len) { diff --git a/lib/zstd_static.h b/lib/zstd_static.h index fa285ad4c..0effaa7ce 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -65,7 +65,7 @@ extern "C" { #define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MIN 12 #define ZSTD_HASHLOG3_MAX 24 -#define ZSTD_HASHLOG3_MIN 12 +#define ZSTD_HASHLOG3_MIN 2 #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7 From 4f7f79ef9dd03eb128abb9a489eb4daab10766ee Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 7 Mar 2016 16:14:58 +0100 Subject: [PATCH 3/3] fixed update of nextToUpdate --- lib/zstd_opt.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 0c4a80f77..68a3c315a 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -250,8 +250,8 @@ static U32 ZSTD_insertBtAndGetAllMatches ( matches[mnum].off = current - matchIndex3; matches[mnum].len = (U32)currentMl; mnum++; - if (currentMl > ZSTD_OPT_NUM) return mnum; - if (ip+currentMl == iLimit) return mnum; /* best possible, and avoid read overflow*/ + if (currentMl > ZSTD_OPT_NUM) goto update; + if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/ } } } @@ -316,6 +316,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( *smallerPtr = *largerPtr = 0; +update: zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; return mnum; } @@ -340,8 +341,8 @@ static U32 ZSTD_BtGetAllMatches_selectMLS ( { switch(matchLengthSearch) { - default : case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches); + default : case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches); case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches); case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches); @@ -367,8 +368,8 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( { switch(matchLengthSearch) { - default : case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches); + default : case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches); case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches); case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches);