minor behavior refinements

specifically, there is no obligation to start streaming compression with pos=0.
stableSrc mode is now compatible with this setup.
This commit is contained in:
Yann Collet
2022-01-26 10:31:24 -08:00
parent 37b87add7a
commit 27d336b099
3 changed files with 21 additions and 15 deletions
+8 -4
View File
@@ -5630,6 +5630,8 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
return 0; return 0;
} }
/* @return provides a minimum amount of data remaining to be flushed from internal buffers
*/
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
ZSTD_outBuffer* output, ZSTD_outBuffer* output,
ZSTD_inBuffer* input, ZSTD_inBuffer* input,
@@ -5644,13 +5646,15 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
/* transparent initialization stage */ /* transparent initialization stage */
if (cctx->streamStage == zcss_init) { if (cctx->streamStage == zcss_init) {
size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */
if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
&& (endOp == ZSTD_e_continue) /* more to come */ && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */
&& (input->pos < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ && (inputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */
/* just wait, allows lazy adaptation of compression parameters */
cctx->expectedInBuffer = *input; cctx->expectedInBuffer = *input;
return (ZSTD_BLOCKSIZE_MAX - input->pos); /* don't do anything : allows lazy adaptation of compression parameters */ return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */
} }
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, inputSize), "CompressStream2 initialization failed");
ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
} }
/* end of transparent initialization stage */ /* end of transparent initialization stage */
+1 -1
View File
@@ -1851,7 +1851,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* memory. However, compression WILL fail if conditions are not respected. * memory. However, compression WILL fail if conditions are not respected.
* *
* WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
* not be modified during compression or it sill result in data corruption. * not be modified during compression or it will result in data corruption.
* This is because zstd needs to reference data in the ZSTD_inBuffer to find * This is because zstd needs to reference data in the ZSTD_inBuffer to find
* matches. Normally zstd maintains its own window buffer for this purpose, * matches. Normally zstd maintains its own window buffer for this purpose,
* but passing this flag tells zstd to rely on user provided buffer instead. * but passing this flag tells zstd to rely on user provided buffer instead.
+12 -10
View File
@@ -873,20 +873,21 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++); DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
CHECK_Z( ZSTD_initCStream(cctx, 1) ); CHECK_Z( ZSTD_initCStream(cctx, 1) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
in.src = CNBuffer; { ZSTD_inBuffer inBuf;
in.size = 100; ZSTD_outBuffer outBuf;
in.pos = 0; inBuf.src = CNBuffer;
{ ZSTD_outBuffer outBuf; inBuf.size = 100;
inBuf.pos = 0;
outBuf.dst = (char*)(compressedBuffer)+cSize; outBuf.dst = (char*)(compressedBuffer)+cSize;
outBuf.size = ZSTD_compressBound(500); outBuf.size = ZSTD_compressBound(500);
outBuf.pos = 0; outBuf.pos = 0;
CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &in) ); CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
in.size = 200; inBuf.size = 200;
CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &in) ); CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
CHECK_Z( ZSTD_flushStream(cctx, &outBuf) ); CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
in.size = 300; inBuf.size = 300;
CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &in) ); CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
if (ZSTD_endStream(cctx, &outBuf) != 0) goto _output_error; /* error, or some data not flushed */ CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@@ -902,6 +903,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
in.src = CNBuffer;
in.pos = out.pos = 0; in.pos = out.pos = 0;
in.size = MIN(CNBufferSize, 10); in.size = MIN(CNBufferSize, 10);
out.size = compressedBufferSize; out.size = compressedBufferSize;