ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed a pretty complex bug when combining ldm + btultra

Browse Source
This commit is contained in:
Yann Collet
2018-05-17 16:13:53 -07:00
parent 134388ba6b
commit 8572b4d09f
3 changed files with 24 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress_internal.h
+6 -4
View File
@@ -580,12 +580,13 @@ MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
{ {
U32 const current = (U32)((BYTE const*)srcEnd - window->base); U32 const current = (U32)((BYTE const*)srcEnd - window->base);
U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0; U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0;
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: current=%u, maxDist=%u", current, maxDist);
if (current > maxDist + loadedDictEnd) { if (current > maxDist + loadedDictEnd) {
U32 const newLowLimit = current - maxDist; U32 const newLowLimit = current - maxDist;
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
if (window->dictLimit < window->lowLimit) { if (window->dictLimit < window->lowLimit) {
DEBUGLOG(5, "Update dictLimit from %u to %u", window->dictLimit, DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
window->lowLimit); window->dictLimit, window->lowLimit);
window->dictLimit = window->lowLimit; window->dictLimit = window->lowLimit;
} }
if (loadedDictEndPtr) if (loadedDictEndPtr)
@@ -605,12 +606,12 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
{ {
BYTE const* const ip = (BYTE const*)src; BYTE const* const ip = (BYTE const*)src;
U32 contiguous = 1; U32 contiguous = 1;
DEBUGLOG(5, "ZSTD_window_update");
/* Check if blocks follow each other */ /* Check if blocks follow each other */
if (src != window->nextSrc) { if (src != window->nextSrc) {
/* not contiguous */ /* not contiguous */
size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
window->dictLimit);
window->lowLimit = window->dictLimit; window->lowLimit = window->dictLimit;
assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
window->dictLimit = (U32)distanceFromBase; window->dictLimit = (U32)distanceFromBase;
@@ -627,6 +628,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
window->lowLimit = lowLimitMax; window->lowLimit = lowLimitMax;
DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
} }
return contiguous; return contiguous;
} }
lib/compress/zstd_ldm.c
+3 -7
View File
@@ -225,12 +225,10 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
{ {
case ZSTD_fast: case ZSTD_fast:
ZSTD_fillHashTable(ms, cParams, iend, ZSTD_dtlm_fast); ZSTD_fillHashTable(ms, cParams, iend, ZSTD_dtlm_fast);
ms->nextToUpdate = (U32)(iend - ms->window.base);
break; break;
case ZSTD_dfast: case ZSTD_dfast:
ZSTD_fillDoubleHashTable(ms, cParams, iend, ZSTD_dtlm_fast); ZSTD_fillDoubleHashTable(ms, cParams, iend, ZSTD_dtlm_fast);
ms->nextToUpdate = (U32)(iend - ms->window.base);
break; break;
case ZSTD_greedy: case ZSTD_greedy:
@@ -597,13 +595,13 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
unsigned const minMatch = cParams->searchLength; unsigned const minMatch = cParams->searchLength;
ZSTD_blockCompressor const blockCompressor = ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, extDict); ZSTD_selectBlockCompressor(cParams->strategy, extDict);
BYTE const* const base = ms->window.base;
/* Input bounds */ /* Input bounds */
BYTE const* const istart = (BYTE const*)src; BYTE const* const istart = (BYTE const*)src;
BYTE const* const iend = istart + srcSize; BYTE const* const iend = istart + srcSize;
/* Input positions */ /* Input positions */
BYTE const* ip = istart; BYTE const* ip = istart;
DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
assert(rawSeqStore->pos <= rawSeqStore->size); assert(rawSeqStore->pos <= rawSeqStore->size);
assert(rawSeqStore->size <= rawSeqStore->capacity); assert(rawSeqStore->size <= rawSeqStore->capacity);
/* Loop through each sequence and apply the block compressor to the lits */ /* Loop through each sequence and apply the block compressor to the lits */
@@ -623,12 +621,12 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_ldm_limitTableUpdate(ms, ip); ZSTD_ldm_limitTableUpdate(ms, ip);
ZSTD_ldm_fillFastTables(ms, cParams, ip); ZSTD_ldm_fillFastTables(ms, cParams, ip);
/* Run the block compressor */ /* Run the block compressor */
DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
{ {
size_t const newLitLength = size_t const newLitLength =
blockCompressor(ms, seqStore, rep, cParams, ip, blockCompressor(ms, seqStore, rep, cParams, ip,
sequence.litLength); sequence.litLength);
ip += sequence.litLength; ip += sequence.litLength;
ms->nextToUpdate = (U32)(ip - base);
/* Update the repcodes */ /* Update the repcodes */
for (i = ZSTD_REP_NUM - 1; i > 0; i--) for (i = ZSTD_REP_NUM - 1; i > 0; i--)
rep[i] = rep[i-1]; rep[i] = rep[i-1];
@@ -644,10 +642,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_ldm_limitTableUpdate(ms, ip); ZSTD_ldm_limitTableUpdate(ms, ip);
ZSTD_ldm_fillFastTables(ms, cParams, ip); ZSTD_ldm_fillFastTables(ms, cParams, ip);
/* Compress the last literals */ /* Compress the last literals */
{ { size_t const lastLiterals = blockCompressor(ms, seqStore, rep, cParams,
size_t const lastLiterals = blockCompressor(ms, seqStore, rep, cParams,
ip, iend - ip); ip, iend - ip);
ms->nextToUpdate = (U32)(iend - base);
return lastLiterals; return lastLiterals;
} }
} }
lib/compress/zstd_opt.c
+15 -4
View File
@@ -481,7 +481,7 @@ void ZSTD_updateTree_internal(
const BYTE* const base = ms->window.base; const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base); U32 const target = (U32)(ip - base);
U32 idx = ms->nextToUpdate; U32 idx = ms->nextToUpdate;
DEBUGLOG(8, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)", DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)",
idx, target, extDict); idx, target, extDict);
while(idx < target) while(idx < target)
@@ -529,7 +529,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
U32 nbCompares = 1U << cParams->searchLog; U32 nbCompares = 1U << cParams->searchLog;
size_t bestLength = lengthToBeat-1; size_t bestLength = lengthToBeat-1;
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches"); DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current);
/* check repCode */ /* check repCode */
{ U32 const lastR = ZSTD_REP_NUM + ll0; { U32 const lastR = ZSTD_REP_NUM + ll0;
@@ -1036,17 +1036,28 @@ size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize) const ZSTD_compressionParameters* cParams, const void* src, size_t srcSize)
{ {
if (ms->opt.litLengthSum==0) { /* first block */ DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */
U32 tmpRep[ZSTD_REP_NUM]; U32 tmpRep[ZSTD_REP_NUM];
DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics");
assert(ms->nextToUpdate >= ms->window.dictLimit assert(ms->nextToUpdate >= ms->window.dictLimit
&& ms->nextToUpdate <= ms->window.dictLimit + 1); && ms->nextToUpdate <= ms->window.dictLimit + 1);
memcpy(tmpRep, rep, sizeof(tmpRep)); memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); /* generate stats into ms->opt*/ ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); /* generate stats into ms->opt*/
ZSTD_resetSeqStore(seqStore); ZSTD_resetSeqStore(seqStore);
ZSTD_window_update(&ms->window, src, srcSize); /* invalidate first scan from history, since it overlaps perfectly */ /* invalidate first scan from history */
ms->window.base -= srcSize;
ms->window.dictLimit += srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt); ZSTD_upscaleStats(&ms->opt);
} }
DEBUGLOG(5, "base=%p, src=%p, src-base=%zi", ms->window.base, src, (const BYTE*)src - (const BYTE*)ms->window.base);
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/); return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/);
} }