ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed corruption issue

Browse Source
This commit is contained in:
Yann Collet
2017-11-15 13:44:24 -08:00
parent 046ea53bef
commit d7e9805028
2 changed files with 16 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_lazy.c
+5 -38
View File
@@ -50,12 +50,6 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
predictedLarge += (predictedLarge>0); predictedLarge += (predictedLarge>0);
#endif /* ZSTD_C_PREDICT */ #endif /* ZSTD_C_PREDICT */
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 2)
g_debuglog_enable = (current <= 8530000);
#endif
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 8)
g_debuglog_enable = (current == 5202593);
#endif
DEBUGLOG(8, "ZSTD_insertBt1 (%u)", current); DEBUGLOG(8, "ZSTD_insertBt1 (%u)", current);
assert(ip <= iend-8); /* required for h calculation */ assert(ip <= iend-8); /* required for h calculation */
@@ -65,12 +59,6 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 8)
if (current == 5189477) g_debuglog_enable = 1;
#endif
DEBUGLOG(8, "index%8u evaluated during insertion of %u (presumed min matchLength:%3u) ",
matchIndex, current, (U32)matchLength);
#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ #ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
if (matchIndex == predictedSmall) { if (matchIndex == predictedSmall) {
@@ -91,34 +79,19 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
continue; continue;
} }
#endif #endif
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
assert(matchIndex+matchLength >= dictLimit); /* might be wrong if extDict is incorrectly set to 0 */
match = base + matchIndex; match = base + matchIndex;
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
{ size_t const controlSize = ZSTD_count(ip, match, iend);
if (controlSize < matchLength) {
DEBUGLOG(2, "Warning !! => matchIndex %u while inserting %u within prefix is smaller than minimum expectation (%u<%u) !",
matchIndex, current, (U32)controlSize, (U32)matchLength);
} }
#endif
if (match[matchLength] == ip[matchLength]) if (match[matchLength] == ip[matchLength])
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
} else { } else {
match = dictBase + matchIndex; match = dictBase + matchIndex;
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
{ size_t const controlSize = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);
if (controlSize < matchLength) {
DEBUGLOG(2, "Warning !! => matchIndex %u while inserting %u into _extDict is smaller than minimum expectation (%u<%u) !",
matchIndex, current, (U32)controlSize, (U32)matchLength);
} }
#endif
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit) if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
} }
DEBUGLOG(8, "matchIndex%8u has %u bytes in common with %u ",
matchIndex, (U32)matchLength, current);
if (matchLength > bestLength) { if (matchLength > bestLength) {
bestLength = matchLength; bestLength = matchLength;
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
@@ -126,24 +99,18 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
} }
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
DEBUGLOG(8, "index %u has equal value at length %u as src : cannot determine > or <",
matchIndex, (U32)matchLength);
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
} }
if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
/* match+1 is smaller than current */ /* match is smaller than current */
DEBUGLOG(8, "matchIndex%8u is smaller than %u (%u < %u)",
matchIndex, current, match[matchLength], ip[matchLength]);
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
} else { } else {
/* match is larger than current */ /* match is larger than current */
DEBUGLOG(8, "matchIndex%8u is larger than %u (%u < %u)",
matchIndex, current, match[matchLength], ip[matchLength]);
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */
lib/compress/zstd_opt.c
+11 -97
View File
@@ -231,7 +231,8 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
/*-************************************* /*-*************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
static U32 ZSTD_insertBtAndGetAllMatches ( FORCE_INLINE_TEMPLATE
U32 ZSTD_insertBtAndGetAllMatches (
ZSTD_CCtx* zc, ZSTD_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit, const int extDict, const BYTE* const ip, const BYTE* const iLimit, const int extDict,
U32 nbCompares, U32 const mls, U32 nbCompares, U32 const mls,
@@ -263,14 +264,8 @@ static U32 ZSTD_insertBtAndGetAllMatches (
size_t bestLength = minMatchLen-1; size_t bestLength = minMatchLen-1;
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 8)
g_debuglog_enable = (current == 5202593);
//g_debuglog_enable = (current == 8845622);
//if (current == 12193408) g_debuglog_enable = 1;
#endif
/* check repCode */ /* check repCode */
#if 0 #if 1
if (!extDict) /*static*/ { if (!extDict) /*static*/ {
U32 const lastR = ZSTD_REP_NUM + ll0; U32 const lastR = ZSTD_REP_NUM + ll0;
U32 repCode; U32 repCode;
@@ -324,7 +319,6 @@ static U32 ZSTD_insertBtAndGetAllMatches (
(void)ll0; (void)rep; (void)minMatch; (void)ll0; (void)rep; (void)minMatch;
#endif #endif
#if 1
/* HC3 match finder */ /* HC3 match finder */
if ((mls == 3) /*static*/ && (bestLength < mls)) { if ((mls == 3) /*static*/ && (bestLength < mls)) {
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip);
@@ -353,7 +347,6 @@ static U32 ZSTD_insertBtAndGetAllMatches (
| (ip+mlen == iLimit) ) { /* best possible */ | (ip+mlen == iLimit) ) { /* best possible */
return 1; return 1;
} } } } } } } }
#endif
hashTable[h] = current; /* Update Hash Table */ hashTable[h] = current; /* Update Hash Table */
@@ -362,64 +355,21 @@ static U32 ZSTD_insertBtAndGetAllMatches (
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match; const BYTE* match;
assert(current > matchIndex); assert(current > matchIndex);
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8)
if (matchIndex==5189477) g_debuglog_enable = 1;
DEBUGLOG(8, "candidate match at index:%7u; presumed min matchLength:%3u ",
matchIndex, (U32)matchLength);
#endif
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */
match = base + matchIndex; match = base + matchIndex;
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
{ size_t const controlSize = ZSTD_count(ip, match, iLimit);
if (controlSize < matchLength) {
DEBUGLOG(2, "Warning !! => matchIndex %u while searching %u within prefix is smaller than minimum expectation (%u<%u) !",
matchIndex, current, (U32)controlSize, (U32)matchLength);
} }
#endif
if (match[matchLength] == ip[matchLength]) { if (match[matchLength] == ip[matchLength]) {
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1;
} }
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8)
{ int i;
RAWLOG(8, "orig %7u: ", current);
for (i=0; i<8; i++) RAWLOG(7," %02X ", ip[i]);
RAWLOG(8, " \n");
RAWLOG(8, "match%7u: ", matchIndex);
for (i=0; i<8; i++) RAWLOG(7," %02X ", match[i]);
RAWLOG(8, " \n");
}
#endif
} else { } else {
match = dictBase + matchIndex; match = dictBase + matchIndex;
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
{ size_t const controlSize = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart);
if (controlSize < matchLength) {
DEBUGLOG(2, "Warning !! => matchIndex %u while searching %u into _extDict is smaller than minimum expectation (%u<%u) !",
matchIndex, current, (U32)controlSize, (U32)matchLength);
} }
#endif
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit) if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* prepare for match[matchLength] */ match = base + matchIndex; /* prepare for match[matchLength] */
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8)
if (matchIndex + 8 < dictLimit)
{ int i;
const BYTE* const ptr = dictBase + matchIndex;
RAWLOG(8, "orig %7u: ", current);
for (i=0; i<8; i++) RAWLOG(7," %02X ", ip[i]);
RAWLOG(8, " \n");
RAWLOG(8, "match%7u: ", matchIndex);
for (i=0; i<8; i++) RAWLOG(7," %02X ", ptr[i]);
RAWLOG(8, " \n");
}
#endif
} }
DEBUGLOG(8, "candidate match has length %u (best:%u) ",
(U32)matchLength, (U32)bestLength);
if (matchLength > bestLength) { if (matchLength > bestLength) {
DEBUGLOG(8, "current %u : found longer match of size%3u at distance%7u",
current, (U32)matchLength, current - matchIndex);
assert(matchEndIdx > matchIndex); assert(matchEndIdx > matchIndex);
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
@@ -429,60 +379,23 @@ static U32 ZSTD_insertBtAndGetAllMatches (
mnum++; mnum++;
if (matchLength > ZSTD_OPT_NUM) break; if (matchLength > ZSTD_OPT_NUM) break;
if (ip+matchLength == iLimit) { /* equal : no way to know if inf or sup */ if (ip+matchLength == iLimit) { /* equal : no way to know if inf or sup */
DEBUGLOG(8, "match at index %u has equal value at length %u : cannot determine > or <",
matchIndex, (U32)matchLength);
break; /* drop, to preserve bt consistency (miss a little bit of compression) */ break; /* drop, to preserve bt consistency (miss a little bit of compression) */
} }
} }
if (match[matchLength] < ip[matchLength]) { if (match[matchLength] < ip[matchLength]) {
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8) /* match smaller than current */
DEBUGLOG(8, "match at index %u is smaller than current (%02X<%02X) (pos%u)",
matchIndex, match[matchLength], ip[matchLength], (U32)matchLength);
{ int i;
RAWLOG(8, "index %u: ", matchIndex);
for (i=0; i<18; i++) RAWLOG(7," %02X ", match[i]);
RAWLOG(8, " \n");
}
#endif
*smallerPtr = matchIndex; /* update smaller idx */ *smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8)
DEBUGLOG(8, "selecting larger candidate : ");
{ int i;
const BYTE* const match2 = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
RAWLOG(8, "index %u: ", matchIndex);
for (i=0; i<18; i++) RAWLOG(7," %02X ", match2[i]);
RAWLOG(8, " \n");
}
#endif
} else { } else {
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8)
DEBUGLOG(8, "match at index %u is larger than current (%02X>%02X) (pos%u)",
matchIndex, match[matchLength], ip[matchLength], (U32)matchLength);
{ int i;
RAWLOG(8, "index %u: ", matchIndex);
for (i=0; i<18; i++) RAWLOG(7," %02X ", match[i]);
RAWLOG(8, " \n");
}
#endif
*largerPtr = matchIndex; *largerPtr = matchIndex;
commonLengthLarger = matchLength; commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr; largerPtr = nextPtr;
matchIndex = nextPtr[0]; matchIndex = nextPtr[0];
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=8)
DEBUGLOG(8, "selecting smaller candidate : ");
{ int i;
const BYTE* const match2 = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
RAWLOG(8, "index %u: ", matchIndex);
for (i=0; i<18; i++) RAWLOG(7," %02X ", match2[i]);
RAWLOG(8, " \n");
}
#endif
} } } }
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
@@ -492,7 +405,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
} }
static U32 ZSTD_BtGetAllMatches ( FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iHighLimit, const int extDict, const BYTE* ip, const BYTE* const iHighLimit, const int extDict,
const U32 maxNbAttempts, const U32 matchLengthSearch, const U32 maxNbAttempts, const U32 matchLengthSearch,
@@ -500,7 +413,8 @@ static U32 ZSTD_BtGetAllMatches (
ZSTD_match_t* matches, const U32 minMatchLen) ZSTD_match_t* matches, const U32 minMatchLen)
{ {
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch); if (extDict) ZSTD_updateTree_extDict(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch);
else ZSTD_updateTree(zc, ip, iHighLimit, maxNbAttempts, matchLengthSearch);
switch(matchLengthSearch) switch(matchLengthSearch)
{ {
case 3 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 3, rep, ll0, matches, minMatchLen); case 3 : return ZSTD_insertBtAndGetAllMatches(zc, ip, iHighLimit, extDict, maxNbAttempts, 3, rep, ll0, matches, minMatchLen);