ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed btultra2 usage with prefix

Browse Source 
notably while using multi-threading
This commit is contained in:
Yann Collet
2018-12-10 18:45:03 -08:00
parent ea441ee4ae
commit 5e6aaa3abb
2 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_opt.c
+8 -4
View File
@@ -1086,8 +1086,7 @@ static void ZSTD_initStats_ultra(
assert(ms->opt.litLengthSum == 0); /* first block */ assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->nextToUpdate >= ms->window.dictLimit assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
&& ms->nextToUpdate <= ms->window.dictLimit + 1);
memcpy(tmpRep, rep, sizeof(tmpRep)); memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
@@ -1121,12 +1120,17 @@ size_t ZSTD_compressBlock_btultra2(
/* 2-pass strategy /* 2-pass strategy
* this strategy makes a first pass over first block to collect statistics * this strategy makes a first pass over first block to collect statistics
* and seed next round's statistics with it. * and seed next round's statistics with it.
* After 1st pass, function forgets everything, and starts a new block.
* Consequently, this can only work if no data has been previously loaded in tables,
* aka, no dictionary, no prefix, no ldm preprocessing.
* The compression ratio gain is generally small (~0.5% on first block), * The compression ratio gain is generally small (~0.5% on first block),
* the cost is 2x cpu time on first block. */ * the cost is 2x cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX); assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */ if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
&& (ms->window.dictLimit - ms->nextToUpdate <= 1) /* no prefix (note: intentional overflow, defined as 2-complement) */
) {
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
} }
tests/zstreamtest.c
+1 -1
View File
@@ -1548,7 +1548,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest,
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */ CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} } } } } }