From d9646dcbb5d6fd265bfe19b5376ef8e15cf3cf11 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 14 Nov 2019 19:39:09 -0500 Subject: [PATCH 1/4] Fixed main compression logic changes --- lib/compress/zstd_compress.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 954d8254e..5bb042483 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2572,23 +2572,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; - { int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams); - size_t cSize = 0; + { size_t cSize; + int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams); if (useTargetCBlockSize) { cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); } else { cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, - ip, blockSize, 0 /* frame */); + ip, blockSize, 1 /* frame */); FORWARD_IF_ERROR(cSize); if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); } else { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(op, cBlockHeader24); + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; } } From 75c34684c0e8fedf3b6afffe91f2ad5321835860 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 15 Nov 2019 12:26:48 -0500 Subject: [PATCH 2/4] Modified existing RLE test to take compressed size into account --- tests/fuzzer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 88f3b83f8..e757daaba 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1999,14 +1999,14 @@ static int basicUnitTests(U32 const seed, double compressibility) /* long rle test */ { size_t sampleSize = 0; + size_t expectedCompressedSize = 39; /* two RLE blocks, zstd 1.4.4 */ DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); - RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; - RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2); + memset((char*)CNBuffer+sampleSize, 'A', 96 KB); sampleSize += 96 KB; cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); - if (ZSTD_isError(cSize)) goto _output_error; + if (ZSTD_isError(cSize) || cSize > expectedCompressedSize * 1.1) goto _output_error; { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); if (regenSize!=sampleSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); From db8efbfe7d5ce6df3e09ef2be93eb4234b32f511 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 15 Nov 2019 14:57:44 -0500 Subject: [PATCH 3/4] Updated comment to reflect actual compression behavior --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index e757daaba..7bf84196d 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1999,7 +1999,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* long rle test */ { size_t sampleSize = 0; - size_t expectedCompressedSize = 39; /* two RLE blocks, zstd 1.4.4 */ + size_t expectedCompressedSize = 39; /* block 1: RLE, block 2: compressed, block 3: RLE, zstd 1.4.4 */ DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; From bc3e21578de1788acd0b05f50a5d6bd674a0482c Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 18 Nov 2019 16:39:16 -0500 Subject: [PATCH 4/4] No margin on RLE test size check --- tests/fuzzer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 7bf84196d..e14a4798b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1999,14 +1999,14 @@ static int basicUnitTests(U32 const seed, double compressibility) /* long rle test */ { size_t sampleSize = 0; - size_t expectedCompressedSize = 39; /* block 1: RLE, block 2: compressed, block 3: RLE, zstd 1.4.4 */ + size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; memset((char*)CNBuffer+sampleSize, 'A', 96 KB); sampleSize += 96 KB; cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); - if (ZSTD_isError(cSize) || cSize > expectedCompressedSize * 1.1) goto _output_error; + if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error; { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); if (regenSize!=sampleSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n");