From 146049a1ea7908342ed99e91275739a4e730d8c5 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 28 Sep 2018 12:09:14 -0700 Subject: [PATCH 1/4] [zstreamtest] Add failing test case --- tests/zstreamtest.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index a7af89eba..d78f00683 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1073,6 +1073,45 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced( + dictionary.start, dictionary.filled, + ZSTD_dlm_byRef, ZSTD_dct_fullDict, + ZSTD_getCParams(3, 0, dictionary.filled), + ZSTD_defaultCMem); + ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; + int remainingInput = 256 * 1024; + + ZSTD_CCtx_reset(zc); + CHECK_Z(ZSTD_CCtx_resetParameters(zc)); + CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1)); + /* Write a bunch of 6 byte blocks */ + while (remainingInput > 0) { + const size_t kSmallBlockSize = 6; + char testBuffer[kSmallBlockSize] = "\xAA\xAA\xAA\xAA\xAA\xAA"; + const size_t outStart = out.pos; + ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0}; + + CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush)); + CHECK(in.pos != in.size, "input not fully consumed"); + remainingInput -= kSmallBlockSize; + } + /* Write several very long offset matches into the dictionary */ + for (int offset = 1024; offset >= 0; offset -= 128) { + size_t start = out.pos; + ZSTD_inBuffer in = {dictionary.start + offset, 128, 0}; + ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end; + CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush)); + CHECK(in.pos != in.size, "input not fully consumed"); + } + /* Ensure decompression works */ + CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled)); + + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); From 6391cd103035d594ed31d356418dcf8d0ceef6b2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 28 Sep 2018 12:09:28 -0700 Subject: [PATCH 2/4] [zstd] Fix newly added test case --- lib/compress/zstd_compress.c | 42 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3eb5ceb21..5e0fb371c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2349,13 +2349,15 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, const void* src, size_t srcSize) { ZSTD_matchState_t* const ms = &zc->blockState.matchState; + size_t cSize; DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%zu, dictLimit=%u, nextToUpdate=%u)", dstCapacity, ms->window.dictLimit, ms->nextToUpdate); assert(srcSize <= ZSTD_BLOCKSIZE_MAX); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength); - return 0; /* don't even attempt compression below a certain srcSize */ + cSize = 0; + goto out; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* required for optimal parser to read stats from dictionary */ @@ -2417,27 +2419,27 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, } } /* encode sequences and literals */ - { size_t const cSize = ZSTD_compressSequences(&zc->seqStore, - &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, - &zc->appliedParams, - dst, dstCapacity, - srcSize, zc->entropyWorkspace, zc->bmi2); - if (!ZSTD_isError(cSize) && cSize != 0) { - /* confirm repcodes and entropy tables */ - ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; - zc->blockState.prevCBlock = zc->blockState.nextCBlock; - zc->blockState.nextCBlock = tmp; - } + cSize = ZSTD_compressSequences(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, dstCapacity, + srcSize, zc->entropyWorkspace, zc->bmi2); - /* We check that dictionaries have offset codes available for the first - * block. After the first block, the offcode table might not have large - * enough codes to represent the offsets in the data. - */ - if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - - return cSize; +out: + if (!ZSTD_isError(cSize) && cSize != 0) { + /* confirm repcodes and entropy tables when emitting a compressed block */ + ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; + zc->blockState.prevCBlock = zc->blockState.nextCBlock; + zc->blockState.nextCBlock = tmp; } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; } From 0e7a7f1def5db7b627130d38f25caae507f66de6 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 28 Sep 2018 12:14:24 -0700 Subject: [PATCH 3/4] Fix warnings --- tests/zstreamtest.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index d78f00683..d7c8567d4 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1088,9 +1088,8 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1)); /* Write a bunch of 6 byte blocks */ while (remainingInput > 0) { - const size_t kSmallBlockSize = 6; - char testBuffer[kSmallBlockSize] = "\xAA\xAA\xAA\xAA\xAA\xAA"; - const size_t outStart = out.pos; + char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA"; + const size_t kSmallBlockSize = sizeof(testBuffer); ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0}; CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush)); @@ -1099,7 +1098,6 @@ static int basicUnitTests(U32 seed, double compressibility) } /* Write several very long offset matches into the dictionary */ for (int offset = 1024; offset >= 0; offset -= 128) { - size_t start = out.pos; ZSTD_inBuffer in = {dictionary.start + offset, 128, 0}; ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end; CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush)); From eb4423e7edfeb1074e32c7619dbee2c7f2f6d93f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 28 Sep 2018 14:24:38 -0700 Subject: [PATCH 4/4] Fix another warning --- tests/zstreamtest.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index d7c8567d4..f47451a3c 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1081,6 +1081,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_defaultCMem); ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; int remainingInput = 256 * 1024; + int offset; ZSTD_CCtx_reset(zc); CHECK_Z(ZSTD_CCtx_resetParameters(zc)); @@ -1097,7 +1098,7 @@ static int basicUnitTests(U32 seed, double compressibility) remainingInput -= kSmallBlockSize; } /* Write several very long offset matches into the dictionary */ - for (int offset = 1024; offset >= 0; offset -= 128) { + for (offset = 1024; offset >= 0; offset -= 128) { ZSTD_inBuffer in = {dictionary.start + offset, 128, 0}; ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end; CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));