From ede4f97153324bf640cdc9020eed1a85edd18ad4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 5 Oct 2020 17:37:19 -0700 Subject: [PATCH] [zstdmt] Fix bug where extra empty blocks are emitted When zstdmt cannot get a buffer and `ZSTD_e_end` is passed an empty compression job can be created. Additionally, `mtctx->frameEnded` can be set to 1, which could potentially cause problems like unterminated blocks. The fix is to adjust to `ZSTD_e_flush` even when we can't get a buffer. --- lib/compress/zstdmt_compress.c | 12 ++++++++++-- tests/zstreamtest.c | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 6f3e06229..98c4ce0bb 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1773,8 +1773,16 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, mtctx->inBuff.filled += syncPoint.toLoad; forwardInputProgress = syncPoint.toLoad>0; } - if ((input->pos < input->size) && (endOp == ZSTD_e_end)) - endOp = ZSTD_e_flush; /* can't end now : not all input consumed */ + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - Input buffer is NULL & empty + * - We filled the input buffer + * - We hit a synchronization point + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; } if ( (mtctx->jobReady) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index e4b9f0dec..13c1530cd 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -2048,7 +2048,11 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, } } /* Enable rsyncable mode 1 in 4 times. */ - setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI); + { + int const rsyncable = (FUZ_rand(&lseed) % 4 == 0); + DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable); + setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI); + } if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );