ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2153 from terrelln/ldm-dict-reset

Browse Source 
[ldm][mt] Fix loadedDictEnd
This commit is contained in:
Yann Collet
2020-05-19 17:41:51 -07:00
committed by GitHub
parent 0dcd3eec43 651d3d73e0
commit 17226f5919
2 changed files with 44 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+1 -3
View File
@@ -513,6 +513,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
memset(serialState->ldmState.bucketOffsets, 0, bucketSize); memset(serialState->ldmState.bucketOffsets, 0, bucketSize);
/* Update window state and fill hash table with dict */ /* Update window state and fill hash table with dict */
serialState->ldmState.loadedDictEnd = 0;
if (dictSize > 0) { if (dictSize > 0) {
if (dictContentType == ZSTD_dct_rawContent) { if (dictContentType == ZSTD_dct_rawContent) {
BYTE const* const dictEnd = (const BYTE*)dict + dictSize; BYTE const* const dictEnd = (const BYTE*)dict + dictSize;
@@ -528,9 +529,6 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
serialState->ldmWindow = serialState->ldmState.window; serialState->ldmWindow = serialState->ldmState.window;
} }
if (params.ldmParams.enableLdm && dict) {
}
serialState->params = params; serialState->params = params;
serialState->params.jobSize = (U32)jobSize; serialState->params.jobSize = (U32)jobSize;
return 0; return 0;
tests/fuzzer.c
+43 -19
View File
@@ -639,7 +639,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, ZSTD_c_nbWorkers)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
@@ -665,7 +665,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const dictSize = kWindowSize * 10; size_t const dictSize = kWindowSize * 10;
size_t const srcSize1 = kWindowSize / 2; size_t const srcSize1 = kWindowSize / 2;
size_t const srcSize2 = kWindowSize * 10; size_t const srcSize2 = kWindowSize * 10;
int nbWorkers;
if (CNBuffSize < dictSize) goto _output_error; if (CNBuffSize < dictSize) goto _output_error;
@@ -680,24 +679,49 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 12)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { /* Round trip once with a dictionary. */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
/* Round trip once with a dictionary. */ cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); CHECK_Z(cSize);
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
CHECK_Z(cSize); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); /* Streaming decompression to catch out of bounds offsets. */
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); {
/* Streaming decompression to catch out of bounds offsets. */ ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
{ ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; CHECK_Z(dSize);
size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); if (dSize != 0) goto _output_error;
CHECK_Z(dSize); }
if (dSize != 0) goto _output_error;
} CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
/* Round trip once with a dictionary. */
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
{
ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
cSize = out.pos;
}
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
{
ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
cSize = out.pos;
}
/* Streaming decompression to catch out of bounds offsets. */
{
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
CHECK_Z(dSize);
if (dSize != 0) goto _output_error;
} }
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);