From 3cc227e90e446f1d195f62038ae50d3e9aeadab2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 19 May 2020 15:55:03 -0700 Subject: [PATCH 1/2] [ldm][mt] Fix loadedDictEnd --- lib/compress/zstdmt_compress.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 5be7edfeb..1e3c8fdbe 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -513,6 +513,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, memset(serialState->ldmState.bucketOffsets, 0, bucketSize); /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; if (dictSize > 0) { if (dictContentType == ZSTD_dct_rawContent) { BYTE const* const dictEnd = (const BYTE*)dict + dictSize; @@ -528,9 +529,6 @@ ZSTDMT_serialState_reset(serialState_t* serialState, serialState->ldmWindow = serialState->ldmState.window; } - if (params.ldmParams.enableLdm && dict) { - } - serialState->params = params; serialState->params.jobSize = (U32)jobSize; return 0; From 651d3d73e0878224923be30c8b9fe9f9b3e01b14 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 19 May 2020 16:14:14 -0700 Subject: [PATCH 2/2] [test] Update the ldm loadedDictEnd test to cover zstdmt --- tests/fuzzer.c | 62 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 49ea53f7f..8ac2864f3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -639,7 +639,7 @@ static int basicUnitTests(U32 const seed, double compressibility) RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, ZSTD_c_nbWorkers)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); @@ -665,7 +665,6 @@ static int basicUnitTests(U32 const seed, double compressibility) size_t const dictSize = kWindowSize * 10; size_t const srcSize1 = kWindowSize / 2; size_t const srcSize2 = kWindowSize * 10; - int nbWorkers; if (CNBuffSize < dictSize) goto _output_error; @@ -680,24 +679,49 @@ static int basicUnitTests(U32 const seed, double compressibility) CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 12)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3)); - for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); - /* Round trip once with a dictionary. */ - CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); - cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); - CHECK_Z(cSize); - CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); - cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); - /* Streaming decompression to catch out of bounds offsets. */ - { - ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; - ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; - size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); - CHECK_Z(dSize); - if (dSize != 0) goto _output_error; - } + /* Round trip once with a dictionary. */ + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); + /* Streaming decompression to catch out of bounds offsets. */ + { + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; + size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); + CHECK_Z(dSize); + if (dSize != 0) goto _output_error; + } + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); + /* Round trip once with a dictionary. */ + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); + { + ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; + ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + cSize = out.pos; + } + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + { + ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; + ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + cSize = out.pos; + } + /* Streaming decompression to catch out of bounds offsets. */ + { + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; + size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); + CHECK_Z(dSize); + if (dSize != 0) goto _output_error; } ZSTD_freeCCtx(cctx);