diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b624e3c18..a674cea40 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2562,6 +2562,14 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + /* This bool is set if there is enough room to output all noCompress superblocks. + * Just checks if the number of compressed blocks we can fit in dstCapacity is + * greater than the optimistic number of blocks we still have remaining. + * This might be UNset when data is uncompressable and we're streaming. */ + + const int enoughDstCapacityForNoCompressSuperBlocks = + (dstCapacity / (blockSize + 7 /* header + checksum */)) > (srcSize / blockSize); assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); @@ -2586,7 +2594,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, { size_t cSize; int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams); - if (useTargetCBlockSize) { + if (useTargetCBlockSize && enoughDstCapacityForNoCompressSuperBlocks) { cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); } else { diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 56936e9f0..337455d9f 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -489,6 +489,54 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++) + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; + size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize); + + /* This is the number of bytes to stream before ending. This value + * was obtained by trial and error :/. */ + + const size_t streamCompressThreshold = 161792; + const size_t streamCompressDelta = 1024; + + /* The first 1/3 of the buffer is compressible and the last 2/3 is + * uncompressible. This is an approximation of the type of data + * the fuzzer generated to catch this bug. Streams like this were making + * zstd generate noCompress superblocks (which are larger than the src + * they come from). Do this enough times, and we'll run out of room + * and throw a dstSize_tooSmall error. */ + + const size_t compressiblePartSize = srcSize/3; + const size_t uncompressiblePartSize = srcSize-compressiblePartSize; + RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed); + RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed); + + /* Setting target block size so that superblock is used */ + + assert(cctx != NULL); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); + + { size_t read; + for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { + ZSTD_inBuffer in = {src, streamCompressDelta, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + assert(!ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue))); + assert(!ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end))); + src += streamCompressDelta; srcSize -= streamCompressDelta; + dst += out.pos; dstCapacity -= out.pos;}} + + /* This is trying to catch a dstSize_tooSmall error */ + + { ZSTD_inBuffer in = {src, srcSize, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + assert(!ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)));} + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) { /* This tests whether or not we leave enough room for the checksum at the end @@ -501,7 +549,7 @@ static int basicUnitTests(U32 const seed, double compressibility) ZSTD_freeCCtx(cctx); } DISPLAYLEVEL(3, "OK \n"); - + DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++); { int level = -1; ZSTD_CCtx* cctx = ZSTD_createCCtx();