fixed repcode before lowLimit

lib/zstd_compress.c

@@ -2040,7 +2040,8 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
 size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
     if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong);
-    zc->params.searchLength = MINMATCH;
+    zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */
+    ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength);
     return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0);
 }
 
@@ -2174,11 +2175,13 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
 
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
 {
+    ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
     return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, MAX(128 KB, dictSize)));
 }
 
 size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
 {
+    ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel);
     return ZSTD_compressBegin_advanced(zc, NULL, 0, ZSTD_getParams(compressionLevel, 0));
 }
 
@@ -2258,11 +2261,13 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
 {
+    ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
     return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
 }
 
 size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
 {
+    ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel);
     return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
 }
 

lib/zstd_decompress.c

@@ -159,7 +159,8 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     dctx->dictEnd = NULL;
     dctx->hufTableX4[0] = HufLog;
     dctx->flagStaticTables = 0;
-    dctx->params.searchLength = MINMATCH;
+    dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */
+    ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength);
     return 0;
 }
 
@@ -749,7 +750,7 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
     }
     op += 8; match += 8;
 
-    if (oMatchEnd > oend-12) {
+    if (oMatchEnd > oend-(16-3)) { // 3 = MINMATCH
         if (op < oend_8) {
             ZSTD_wildcopy(op, match, oend_8 - op);
             match += oend_8 - op;
@@ -857,6 +858,8 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
     if (srcSize >= BLOCKSIZE) return ERROR(srcSize_wrong);
 
+    ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength);
+
     /* Decode literals sub-block */
     litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
     if (ZSTD_isError(litCSize)) return litCSize;
@@ -966,6 +969,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                  const void* dict, size_t dictSize)
 {
     ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
+    ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength);
     ZSTD_checkContinuity(dctx, dst);
     return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize);
 }

lib/zstd_internal.h

@@ -51,8 +51,17 @@
 *  Common constants
 ***************************************/
 #define ZSTD_OPT_DEBUG 0     // 1 = tableID=0;  3 = price func tests;  5 = check encoded sequences;  9 = full logs
-#if ZSTD_OPT_DEBUG > 0
+#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>0
-    #include <stdio.h>  /* for debug */
+    #include <stdio.h>
+#endif
+#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
+    #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
+    #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
+    #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__)
+#else
+    #define ZSTD_LOG_PARSER(...)
+    #define ZSTD_LOG_ENCODE(...)
+    #define ZSTD_LOG_BLOCK(...)
 #endif
 
 #define ZSTD_DICT_MAGIC  0xEC30A435

lib/zstd_opt.h

@@ -47,8 +47,7 @@ FORCE_INLINE U32 ZSTD_GETPRICE(seqStore_t* seqStorePtr, U32 litLength, const BYT
     price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit(seqStorePtr->matchLengthSum+1) - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1);
 
 #if ZSTD_OPT_DEBUG >= 3
-    switch (seqStorePtr->priceFunc)
+    switch (seqStorePtr->priceFunc) {
-    {
         default:
         case 0:
             return 1 + price + seqStorePtr->factor + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum));
@@ -557,8 +556,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
                 ml2 = (U32)ZSTD_count(ip, ip-offset, iend);
             else
                 ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend);
-            if (offset >= 8)
+            if ((offset >= 8) && (ml2 < mlen || ml2 < MINMATCHOPT)) {
-            if (ml2 < mlen || ml2 < MINMATCHOPT) {
                 printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
             if (ip < anchor) {
                 printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
@@ -572,7 +570,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
         }   /* for (cur=0; cur < last_pos; ) */
 
         /* check immediate repcode */
-        while ( (anchor <= ilimit)
+        while ((anchor >= prefixStart + rep_2) && (anchor <= ilimit)
              && (MEM_readMINMATCH(anchor) == MEM_readMINMATCH(anchor - rep_2)) ) {
             /* store sequence */
             best_mlen = (U32)ZSTD_count(anchor+MINMATCHOPT, anchor+MINMATCHOPT-rep_2, iend);
@@ -614,7 +612,8 @@ void ZSTD_COMPRESSBLOCK_OPT_EXTDICT_GENERIC(ZSTD_CCtx* ctx,
     const BYTE* const prefixStart = base + dictLimit;
     const BYTE* const dictBase = ctx->dictBase;
     const BYTE* const dictEnd  = dictBase + dictLimit;
-
+    const U32 lowLimit = ctx->lowLimit;
+   
     U32 rep_2=REPCODE_STARTVALUE, rep_1=REPCODE_STARTVALUE;
     const U32 maxSearches = 1U << ctx->params.searchLog;
     const U32 mls = ctx->params.searchLength;
@@ -892,21 +891,16 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
 
 #if ZSTD_OPT_DEBUG >= 5
             U32 ml2;
-            if (offset)
+            if (offset) {
-            {
+                if (offset > (size_t)(ip - prefixStart))  {
-                if (offset > (size_t)(ip - prefixStart)) 
-                {
                     const BYTE* match = dictEnd - (offset - (ip - prefixStart));
                     ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);
                     ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)offset, dictBase, dictEnd, prefixStart, ip, match);
                 }
-                else
+                else ml2 = (U32)ZSTD_count(ip, ip-offset, iend);
-                    ml2 = (U32)ZSTD_count(ip, ip-offset, iend);
             }
-            else
+            else ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend);
-                ml2 = (U32)ZSTD_count(ip, ip-rep_1, iend);
+            if ((offset >= 8) && (ml2 < mlen || ml2 < MINMATCHOPT)) {
-            if (offset >= 8)
-            if (ml2 < mlen || ml2 < MINMATCHOPT) {
                 printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
             if (ip < anchor) {
                 printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
@@ -920,7 +914,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
         }
 
         /* check immediate repcode */
-        while (anchor <= ilimit) {
+        while ((anchor >= base + lowLimit + rep_2) && (anchor <= ilimit)) {
             const U32 repIndex = (U32)((anchor-base) - rep_2);
             const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
             const BYTE* const repMatch = repBase + repIndex;

lib/zstd_opt_internal.h

@@ -45,17 +45,6 @@
 #define ZSTD_FREQ_STEP  1
 #define ZSTD_FREQ_DIV   5
 
-/*-  Debug  -*/
-#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
-    #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
-    #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
-    #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__)
-#else
-    #define ZSTD_LOG_PARSER(...)
-    #define ZSTD_LOG_ENCODE(...)
-    #define ZSTD_LOG_BLOCK(...)
-#endif
-
 
 typedef struct {
     U32 off;