ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for in-place decompression

Browse Source 
* Add a function and macro ZSTD_decompressionMargin() that computes the
  decompression margin for in-place decompression. The function computes
  a tight margin that works in all cases, and the macro computes an upper
  bound that will only work if flush isn't used.
* When doing in-place decompression, make sure that our output buffer
  doesn't overlap with the input buffer. This ensures that we don't
  decide to use the portion of the output buffer that overlaps the input
  buffer for temporary memory, like for literals.
* Add a simple unit test.
* Add in-place decompression to the simple_round_trip and
  stream_round_trip fuzzers. This should help verify that our margin stays
  correct.
This commit is contained in:
Nick Terrell
2023-01-12 16:28:08 -08:00
committed by Nick Terrell
parent 423500d1ae
commit 5b266196a4
7 changed files with 227 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzz/simple_round_trip.c
+37 -7
View File
@@ -26,6 +26,23 @@
static ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_t srcSize, int hasSmallBlocks)
{
size_t margin = ZSTD_decompressionMargin(compressed, cSize);
if (!hasSmallBlocks) {
/* The macro should be correct in this case, but it may be smaller
* because of e.g. block splitting, so take the smaller of the two.
*/
ZSTD_frameHeader zfh;
size_t marginM;
FUZZ_ZASSERT(ZSTD_getFrameHeader(&zfh, compressed, cSize));
marginM = ZSTD_DECOMPRESSION_MARGIN(srcSize, zfh.blockSizeMax);
if (marginM < margin)
margin = marginM;
}
return margin;
}
static size_t roundTripTest(void *result, size_t resultCapacity,
void *compressed, size_t compressedCapacity,
const void *src, size_t srcSize,
@@ -67,6 +84,25 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
}
dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
FUZZ_ZASSERT(dSize);
FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, result, dSize), "Corruption!");
{
size_t margin = getDecompressionMargin(compressed, cSize, srcSize, targetCBlockSize);
size_t const outputSize = srcSize + margin;
char* const output = (char*)FUZZ_malloc(outputSize);
char* const input = output + outputSize - cSize;
FUZZ_ASSERT(outputSize >= cSize);
memcpy(input, compressed, cSize);
dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
FUZZ_ZASSERT(dSize);
FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, srcSize), "Corruption!");
free(output);
}
/* When superblock is enabled make sure we don't expand the block more than expected.
* NOTE: This test is currently disabled because superblock mode can arbitrarily
* expand the block in the worst case. Once superblock mode has been improved we can
@@ -120,13 +156,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
FUZZ_ASSERT(dctx);
}
{
size_t const result =
roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
FUZZ_ZASSERT(result);
FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
}
roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
free(rBuf);
free(cBuf);
FUZZ_dataProducer_free(producer);
tests/fuzz/stream_round_trip.c
+18
View File
@@ -166,6 +166,24 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
FUZZ_ZASSERT(rSize);
FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
/* Test in-place decompression (note the macro doesn't work in this case) */
{
size_t const margin = ZSTD_decompressionMargin(cBuf, cSize);
size_t const outputSize = size + margin;
char* const output = (char*)FUZZ_malloc(outputSize);
char* const input = output + outputSize - cSize;
size_t dSize;
FUZZ_ASSERT(outputSize >= cSize);
memcpy(input, cBuf, cSize);
dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
FUZZ_ZASSERT(dSize);
FUZZ_ASSERT_MSG(dSize == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, size), "Corruption!");
free(output);
}
}
FUZZ_dataProducer_free(producer);
tests/fuzzer.c
+54
View File
@@ -1220,6 +1220,60 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
CHECK_Z(cSize);
CHECK_LT(CNBuffSize, cSize);
{
size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
size_t const outputSize = (CNBuffSize + margin);
char* output = malloc(outputSize);
char* input = output + outputSize - cSize;
CHECK_LT(cSize, CNBuffSize + margin);
CHECK(output != NULL);
CHECK_Z(margin);
CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
memcpy(input, compressedBuffer, cSize);
{
size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
CHECK_Z(dSize);
CHECK_EQ(dSize, CNBuffSize);
}
CHECK(!memcmp(output, CNBuffer, CNBuffSize));
free(output);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
CHECK_Z(cSize);
{
size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
CHECK_Z(cSize2);
cSize += cSize2;
}
{
size_t const srcSize = (CNBuffSize / 3) * 2;
size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
size_t const outputSize = (CNBuffSize + margin);
char* output = malloc(outputSize);
char* input = output + outputSize - cSize;
CHECK_LT(cSize, CNBuffSize + margin);
CHECK(output != NULL);
CHECK_Z(margin);
memcpy(input, compressedBuffer, cSize);
{
size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
CHECK_Z(dSize);
CHECK_EQ(dSize, srcSize);
}
CHECK(!memcmp(output, CNBuffer, srcSize));
free(output);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();