a bugfix for backward matches

lib/zstd_opt.c

@@ -185,20 +185,22 @@ size_t ZSTD_insertBtAndGetAllMatches (
                         U32 nbCompares, const U32 mls,
                         U32 extDict, ZSTD_match_t* matches, size_t bestLength)
 {
-    U32* const hashTable = zc->hashTable;
+    const BYTE* const base = zc->base;
+    const U32 current = (U32)(ip-base);
     const U32 hashLog = zc->params.hashLog;
     const size_t h  = ZSTD_hashPtr(ip, hashLog, mls);
+    U32* const hashTable = zc->hashTable;
+    U32 matchIndex  = hashTable[h];
+    if (matchIndex >= current) return 0;
+
     U32* const bt   = zc->contentTable;
     const U32 btLog = zc->params.contentLog - 1;
     const U32 btMask= (1 << btLog) - 1;
-    U32 matchIndex  = hashTable[h];
     size_t commonLengthSmaller=0, commonLengthLarger=0;
-    const BYTE* const base = zc->base;
     const BYTE* const dictBase = zc->dictBase;
     const U32 dictLimit = zc->dictLimit;
     const BYTE* const dictEnd = dictBase + dictLimit;
     const BYTE* const prefixStart = base + dictLimit;
-    const U32 current = (U32)(ip-base);
     const U32 btLow = btMask >= current ? 0 : current - btMask;
     const U32 windowLow = zc->lowLimit;
     U32* smallerPtr = bt + 2*(current&btMask);
@@ -285,6 +287,8 @@ FORCE_INLINE size_t ZSTD_BtGetAllMatches_selectMLS (
                         const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit,
                         const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, size_t minml)
 {
+    if (iLowLimit) {}; // skip warnings
+
     switch(matchLengthSearch)
     {
     default :
@@ -320,6 +324,7 @@ size_t ZSTD_HcGetAllMatches_generic (
 
     /* HC4 match finder */
     matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
+    if (matchIndex >= current) return 0;
 
     while ((matchIndex>lowLimit) && (nbAttempts)) {
         size_t currentMl=0;
@@ -329,7 +334,6 @@ size_t ZSTD_HcGetAllMatches_generic (
             match = base + matchIndex;
             if (match[minml] == ip[minml])   /* potentially better */
                 currentMl = ZSTD_count(ip, match, iHighLimit);
-
             if (currentMl > 0) {
                 while ((match-back > base) && (ip-back > iLowLimit) && (ip[-back-1] == match[-back-1])) back++; /* backward match extension */
                 currentMl += back;
@@ -338,6 +342,10 @@ size_t ZSTD_HcGetAllMatches_generic (
             match = dictBase + matchIndex;
             if (MEM_read32(match) == MEM_read32(ip))   /* assumption : matchIndex <= dictLimit-4 (by table construction) */
                 currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, prefixStart) + MINMATCH;
+            if (currentMl > 0) {
+                while ((match-back > dictBase) && (ip-back > iLowLimit) && (ip[-back-1] == match[-back-1])) back++; /* backward match extension */
+                currentMl += back;
+            }
         }
 
         /* save best solution */
@@ -576,19 +584,22 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
 
 
 
-           if (!opt[cur].off && opt[cur].mlen != 1) {
+           size_t cur_rep;
-                mlen = ZSTD_count(inr, inr - opt[cur].rep2, iend); // check rep2
-                ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), opt[cur].rep2, mlen);   
-           }
-           else {
-                mlen = ZSTD_count(inr, inr - opt[cur].rep, iend); // check rep
-                ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), opt[cur].rep, mlen);   
-           }
-
            best_mlen = 0;
 
-           if (mlen >= MINMATCH)
+           if (!opt[cur].off && opt[cur].mlen != 1) {
+               cur_rep = opt[cur].rep2;
+               ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);   
+           }
+           else {
+               cur_rep = opt[cur].rep;
+               ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen);   
+           }
+
+
+           if (MEM_read32(inr) == MEM_read32(inr - cur_rep)) // check rep
            {
+              mlen = ZSTD_count(inr+MINMATCH, inr+MINMATCH - cur_rep, iend) + MINMATCH; 
               ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off);
 
               if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM)
@@ -666,6 +677,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                 best_mlen = (cur2 + matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM - cur2;
 
                 ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos);
+                if (mlen < matches[i].back + 1)
+                    mlen = matches[i].back + 1;
 
                 while (mlen <= best_mlen)
                 {
@@ -739,7 +752,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
 
 
             size_t litLength = ip - anchor;
-            ZSTD_LOG_ENCODE("%d/%d: BEFORE_ENCODE literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
+            ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
 
             if (offset)
             {
@@ -756,7 +769,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
                 }
             }
 
-            ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
+            ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2);
        //     printf("orig="); print_hex_text(ip, mlen, 0);
        //     printf("match="); print_hex_text(ip-offset, mlen, 0);
 
@@ -766,33 +779,18 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
                 ml2 = ZSTD_count(ip, ip-offset, iend);
             else
                 ml2 = ZSTD_count(ip, ip-rep_1, iend);
-
+            if (ml2 < mlen && ml2 < MINMATCH) {
-            if (ml2 < mlen && ml2 < MINMATCH)
+                printf("%d: ERROR iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
-            {
+            if (ip < anchor) {
-                printf("%d: ERROR iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2);
+                printf("%d: ERROR ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-                exit(0);
+            if (ip - offset < ctx->base) {
-            }
+                printf("%d: ERROR ip - offset < base iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-
+            if ((int)offset >= (1 << ctx->params.windowLog)) {
-            if (ip < anchor)
+                printf("%d: offset >= (1 << params.windowLog) iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-            {
+            if (mlen < MINMATCH) {
-                printf("%d: ERROR ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset);
+                printf("%d: ERROR mlen < MINMATCH iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-                exit(0);
+            if (ip + mlen > iend) {
-            }
+                printf("%d: ERROR ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-            if (ip - offset < base)
-            {
-                printf("%d: ERROR ip - offset < base iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset);
-                exit(0);
-            }
-            if (mlen < MINMATCH)
-            {
-                printf("%d: ERROR mlen < MINMATCH iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset);
-                exit(0);
-            }
-            if (ip + mlen > iend) 
-            {
-                printf("%d: ERROR ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset);
-                exit(0);
-            }
 #endif
 
             ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);