ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2094 from terrelln/stable-dst

Browse Source 
[lib] Add ZSTD_d_stableOutBuffer + fix single-pass mode for empty frames
This commit is contained in:
Nick Terrell
2020-04-28 17:53:24 -07:00
committed by GitHub
parent f17ac423b2 55a57d46be
commit 0ed07f6dfe
7 changed files with 256 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzz/stream_decompress.c
+13 -1
View File
@@ -70,6 +70,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
* buffers in a row. */
int prevInWasZero = 0;
int prevOutWasZero = 0;
int stableOutBuffer;
ZSTD_outBuffer out;
size = FUZZ_dataProducer_reserveDataPrefix(producer);
/* Allocate all buffers and contexts if not already allocated */
@@ -85,11 +87,21 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
FUZZ_ZASSERT(ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only));
}
stableOutBuffer = FUZZ_dataProducer_uint32Range(producer, 0, 10) == 5;
if (stableOutBuffer) {
FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dstream, ZSTD_d_stableOutBuffer, 1));
out.dst = buf;
out.size = kBufSize;
out.pos = 0;
}
while (size > 0) {
ZSTD_inBuffer in = makeInBuffer(&src, &size, producer, prevInWasZero ? 1 : 0);
prevInWasZero = in.size == 0;
while (in.pos != in.size) {
ZSTD_outBuffer out = makeOutBuffer(producer, prevOutWasZero ? 1 : 0);
if (!stableOutBuffer || FUZZ_dataProducer_uint32Range(producer, 0, 100) == 55) {
out = makeOutBuffer(producer, prevOutWasZero ? 1 : 0);
}
prevOutWasZero = out.size == 0;
size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
if (ZSTD_isError(rc)) goto error;
tests/zstreamtest.c
+112
View File
@@ -641,6 +641,118 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
/* Decompression single pass with empty frame */
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
CHECK_Z(cSize);
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
{ ZSTD_DCtx* dctx = ZSTD_createDCtx();
size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
outBuff.dst = decodedBuffer;
outBuff.pos = 0;
outBuff.size = CNBufferSize;
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
CHECK_Z(r);
CHECK(r != 0, "Entire frame must be decompressed");
CHECK(outBuff.pos != 0, "Wrong size!");
CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
}
CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
/* Decompression with ZSTD_d_stableOutBuffer */
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
CHECK_Z(cSize);
{ ZSTD_DCtx* dctx = ZSTD_createDCtx();
size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
size_t dctxSize1;
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
outBuff.dst = decodedBuffer;
outBuff.pos = 0;
outBuff.size = CNBufferSize;
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
CHECK_Z(r);
CHECK(r != 0, "Entire frame must be decompressed");
CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
}
CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
outBuff.pos = 0;
inBuff.pos = 0;
inBuff.size = 0;
while (inBuff.pos < cSize) {
inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
}
CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
dctxSize1 = ZSTD_sizeof_DCtx(dctx);
CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
outBuff.pos = 0;
outBuff.size = CNBufferSize - 1;
{ size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
inBuff.src = compressedBuffer;
inBuff.size = cSize - 1;
inBuff.pos = 0;
outBuff.pos = 0;
outBuff.size = CNBufferSize;
CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
++inBuff.size;
outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
outBuff.pos = 0;
inBuff.pos = 0;
inBuff.size = 0;
while (inBuff.pos < cSize) {
inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
}
CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeDCtx(dctx);
}
/* CDict scenario */
DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );