ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move stream orchestration into Rust

Browse Source 
The high-level single-thread stream path previously combined stable-input
rewind, buffer fill, direct-versus-buffered output, pending flush, end-of-frame
reset, and progress hints in C. The multithreaded path likewise owned input
range selection, rsync and end-directive adjustments, job creation decisions,
and outer flush return policy in C.

Add explicit C/Rust projections and callbacks. Rust now drives the
single-thread state machine and the MT scheduling and flush decision sequence,
while C retains CCtx-sensitive block callbacks, reusable buffers, job
descriptors, synchronization, and worker lifecycle. Focused tests cover stable
and buffered stream behavior, errors, pending output, frame completion, and MT
scheduler branches. Sequence-store and dictionary boundaries remain unchanged.

Test Plan:
- Rust lib and CLI checks, tests, formatting, and clippy passed.
- Rust legacy feature matrix passed: 556 tests.
- Native lib and CLI builds passed; CLI tests passed: 41 tests.
- Native test-zstd, fuzzer, zstream, and decode-corpus gates passed.
This commit is contained in:
ddidderr
2026-07-19 00:14:18 +02:00
parent e3d8f61ec5
commit dbc6992f3b
4 changed files with 1621 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+115 -179
View File
@@ -200,6 +200,78 @@ typedef char ZSTD_rust_compress_continue_state_layout[
&& sizeof(ZSTD_rust_compressContinueState)
== (sizeof(void*) == 8 ? 96 : 52))
? 1 : -1];
/* Rust owns the single-threaded buffered/stable stream state machine. The
* projection contains only stream bookkeeping and callback slots; operations
* which still need the private CCtx layout remain C callbacks. */
typedef size_t (*ZSTD_rust_compressStreamBlock_f)(
void* context, void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
typedef size_t (*ZSTD_rust_compressStreamReset_f)(void* context);
typedef struct {
void* callbackContext;
int inBufferMode;
int outBufferMode;
ZSTD_cStreamStage* streamStage;
size_t blockSizeMax;
size_t* stableInNotConsumed;
void* inBuff;
size_t inBuffSize;
size_t* inToCompress;
size_t* inBuffPos;
size_t* inBuffTarget;
void* outBuff;
size_t outBuffSize;
size_t* outBuffContentSize;
size_t* outBuffFlushedSize;
U32* frameEnded;
ZSTD_rust_compressStreamBlock_f compressContinue;
ZSTD_rust_compressStreamBlock_f compressEnd;
ZSTD_rust_compressStreamReset_f resetSession;
} ZSTD_rust_compressStreamState;
size_t ZSTD_rust_compressStreamGeneric(
const ZSTD_rust_compressStreamState* state,
ZSTD_outBuffer* output, ZSTD_inBuffer* input, int flushMode);
typedef char ZSTD_rust_compress_stream_state_layout[
(offsetof(ZSTD_rust_compressStreamState, callbackContext) == 0
&& offsetof(ZSTD_rust_compressStreamState, inBufferMode) == sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamState, outBufferMode)
== sizeof(void*) + sizeof(int)
&& offsetof(ZSTD_rust_compressStreamState, streamStage)
== sizeof(void*) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressStreamState, blockSizeMax)
== 2 * sizeof(void*) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressStreamState, stableInNotConsumed)
== 3 * sizeof(void*) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressStreamState, inBuff)
== 4 * sizeof(void*) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressStreamState, inBuffSize)
== 5 * sizeof(void*) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressStreamState, inToCompress)
== 5 * sizeof(void*) + 2 * sizeof(int) + sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, inBuffPos)
== 6 * sizeof(void*) + 2 * sizeof(int) + sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, inBuffTarget)
== 7 * sizeof(void*) + 2 * sizeof(int) + sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, outBuff)
== 8 * sizeof(void*) + 2 * sizeof(int) + sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, outBuffSize)
== 9 * sizeof(void*) + 2 * sizeof(int) + sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, outBuffContentSize)
== 9 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, outBuffFlushedSize)
== 10 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, frameEnded)
== 11 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, compressContinue)
== 12 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, compressEnd)
== 13 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStreamState, resetSession)
== 14 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t)
&& sizeof(ZSTD_rust_compressStreamState)
== 15 * sizeof(void*) + 2 * sizeof(int) + 2 * sizeof(size_t))
? 1 : -1];
/* The target-sized block body only needs this narrow projection of ZSTD_CCtx.
* Matchfinder/window state, sequence-store construction, and outer repeat-mode
* cleanup remain in C. */
@@ -4498,190 +4570,54 @@ static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
/** ZSTD_compressStream_generic():
* internal function for all *compressStream*() variants
* @return : hint size for next input to complete ongoing block */
static size_t ZSTD_rust_compressStream_continue(
void* context, void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
return ZSTD_compressContinue_public(
(ZSTD_CCtx*)context, dst, dstCapacity, src, srcSize);
}
static size_t ZSTD_rust_compressStream_end(
void* context, void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
return ZSTD_compressEnd_public(
(ZSTD_CCtx*)context, dst, dstCapacity, src, srcSize);
}
static size_t ZSTD_rust_compressStream_reset(void* context)
{
return ZSTD_CCtx_reset((ZSTD_CCtx*)context, ZSTD_reset_session_only);
}
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode)
{
const char* const istart = (assert(input != NULL), (const char*)input->src);
const char* const iend = (istart != NULL) ? istart + input->size : istart;
const char* ip = (istart != NULL) ? istart + input->pos : istart;
char* const ostart = (assert(output != NULL), (char*)output->dst);
char* const oend = (ostart != NULL) ? ostart + output->size : ostart;
char* op = (ostart != NULL) ? ostart + output->pos : ostart;
U32 someMoreWork = 1;
/* check expectations */
DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos);
assert(zcs != NULL);
if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) {
assert(input->pos >= zcs->stableIn_notConsumed);
input->pos -= zcs->stableIn_notConsumed;
if (ip) ip -= zcs->stableIn_notConsumed;
zcs->stableIn_notConsumed = 0;
}
if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
assert(zcs->inBuff != NULL);
assert(zcs->inBuffSize > 0);
}
if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) {
assert(zcs->outBuff != NULL);
assert(zcs->outBuffSize > 0);
}
if (input->src == NULL) assert(input->size == 0);
assert(input->pos <= input->size);
if (output->dst == NULL) assert(output->size == 0);
assert(output->pos <= output->size);
assert((U32)flushMode <= (U32)ZSTD_e_end);
while (someMoreWork) {
switch(zcs->streamStage)
{
case zcss_init:
RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!");
case zcss_load:
if ( (flushMode == ZSTD_e_end)
&& ( (size_t)(oend-op) >= ZSTD_compressBound((size_t)(iend-ip)) /* Enough output space */
|| zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
&& (zcs->inBuffPos == 0) ) {
/* shortcut to compression pass directly into output buffer */
size_t const cSize = ZSTD_compressEnd_public(zcs,
op, (size_t)(oend-op),
ip, (size_t)(iend-ip));
DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
ip = iend;
op += cSize;
zcs->frameEnded = 1;
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
someMoreWork = 0; break;
}
/* complete loading into inBuffer in buffered mode */
if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(
zcs->inBuff + zcs->inBuffPos, toLoad,
ip, (size_t)(iend-ip));
zcs->inBuffPos += loaded;
if (ip) ip += loaded;
if ( (flushMode == ZSTD_e_continue)
&& (zcs->inBuffPos < zcs->inBuffTarget) ) {
/* not enough input to fill full block : stop here */
someMoreWork = 0; break;
}
if ( (flushMode == ZSTD_e_flush)
&& (zcs->inBuffPos == zcs->inToCompress) ) {
/* empty */
someMoreWork = 0; break;
}
} else {
assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
if ( (flushMode == ZSTD_e_continue)
&& ( (size_t)(iend - ip) < zcs->blockSizeMax) ) {
/* can't compress a full block : stop here */
zcs->stableIn_notConsumed = (size_t)(iend - ip);
ip = iend; /* pretend to have consumed input */
someMoreWork = 0; break;
}
if ( (flushMode == ZSTD_e_flush)
&& (ip == iend) ) {
/* empty */
someMoreWork = 0; break;
}
}
/* compress current block (note : this stage cannot be stopped in the middle) */
DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
{ int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered);
void* cDst;
size_t cSize;
size_t oSize = (size_t)(oend-op);
size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress
: MIN((size_t)(iend - ip), zcs->blockSizeMax);
if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
cDst = op; /* compress into output buffer, to skip flush stage */
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
if (inputBuffered) {
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
cSize = lastBlock ?
ZSTD_compressEnd_public(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue_public(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize);
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSizeMax;
if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSizeMax;
DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
(unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
if (!lastBlock)
assert(zcs->inBuffTarget <= zcs->inBuffSize);
zcs->inToCompress = zcs->inBuffPos;
} else { /* !inputBuffered, hence ZSTD_bm_stable */
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
cSize = lastBlock ?
ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) :
ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize);
/* Consume the input prior to error checking to mirror buffered mode. */
if (ip) ip += iSize;
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
if (lastBlock) assert(ip == iend);
}
if (cDst == op) { /* no need to flush */
op += cSize;
if (zcs->frameEnded) {
DEBUGLOG(5, "Frame completed directly in outBuffer");
someMoreWork = 0;
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
}
break;
}
zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0;
zcs->streamStage = zcss_flush; /* pass-through to flush stage */
}
ZSTD_FALLTHROUGH;
case zcss_flush:
DEBUGLOG(5, "flush stage");
assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered);
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op),
zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
(unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
if (flushed)
op += flushed;
zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) {
/* flush not fully completed, presumably because dst is too small */
assert(op==oend);
someMoreWork = 0;
break;
}
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
if (zcs->frameEnded) {
DEBUGLOG(5, "Frame completed on flush");
someMoreWork = 0;
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
break;
}
zcs->streamStage = zcss_load;
break;
}
default: /* impossible */
assert(0);
}
}
input->pos = (size_t)(ip - istart);
output->pos = (size_t)(op - ostart);
if (zcs->frameEnded) return 0;
return ZSTD_nextInputSizeHint(zcs);
ZSTD_rust_compressStreamState state;
state.callbackContext = zcs;
state.inBufferMode = (int)zcs->appliedParams.inBufferMode;
state.outBufferMode = (int)zcs->appliedParams.outBufferMode;
state.streamStage = &zcs->streamStage;
state.blockSizeMax = zcs->blockSizeMax;
state.stableInNotConsumed = &zcs->stableIn_notConsumed;
state.inBuff = zcs->inBuff;
state.inBuffSize = zcs->inBuffSize;
state.inToCompress = &zcs->inToCompress;
state.inBuffPos = &zcs->inBuffPos;
state.inBuffTarget = &zcs->inBuffTarget;
state.outBuff = zcs->outBuff;
state.outBuffSize = zcs->outBuffSize;
state.outBuffContentSize = &zcs->outBuffContentSize;
state.outBuffFlushedSize = &zcs->outBuffFlushedSize;
state.frameEnded = &zcs->frameEnded;
state.compressContinue = ZSTD_rust_compressStream_continue;
state.compressEnd = ZSTD_rust_compressStream_end;
state.resetSession = ZSTD_rust_compressStream_reset;
return ZSTD_rust_compressStreamGeneric(
&state, output, input, (int)flushMode);
}
static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx)
lib/compress/zstdmt_compress.c
+128 -77
View File
@@ -184,6 +184,65 @@ ZSTDMT_RustFlushProducedResult ZSTDMT_rust_flushProduced(
ZSTDMT_flushCompleteJobFn completeJob,
ZSTDMT_flushErrorFn onError);
/* The Rust outer scheduler sees only this scalar snapshot. The MT context,
* reusable input buffer, worker pool, and all synchronization remain private
* to this translation unit. */
typedef struct {
unsigned frameEnded;
unsigned jobReady;
void* inBuffStart;
size_t inBuffCapacity;
size_t inBuffFilled;
size_t targetSectionSize;
int rsyncable;
U64 rsyncPrimePower;
U64 rsyncHitMask;
} ZSTDMT_RustCompressStreamContextProjection;
typedef struct {
void* bufferStart;
size_t bufferCapacity;
size_t bufferFilled;
} ZSTDMT_RustStreamInputRangeProjection;
typedef struct {
const void* src;
size_t size;
size_t pos;
} ZSTDMT_RustStreamInputProjection;
typedef struct {
void* dst;
size_t size;
size_t pos;
} ZSTDMT_RustStreamOutputProjection;
typedef struct {
size_t toLoad;
int flush;
} ZSTDMT_RustSyncPointProjection;
typedef struct {
size_t result;
size_t outputPos;
} ZSTDMT_RustStreamFlushResult;
typedef struct {
size_t result;
size_t inputPos;
size_t outputPos;
} ZSTDMT_RustCompressStreamResult;
typedef int (*ZSTDMT_streamTryGetInputRangeFn)(
void* opaque, ZSTDMT_RustStreamInputRangeProjection* projection);
typedef int (*ZSTDMT_streamLoadInputFn)(void* opaque, const void* src, size_t size);
typedef size_t (*ZSTDMT_streamCreateJobFn)(void* opaque, size_t srcSize, unsigned end);
typedef ZSTDMT_RustStreamFlushResult (*ZSTDMT_streamFlushProducedFn)(
void* opaque, void* outputDst, size_t outputSize, size_t outputPos,
unsigned blockToFlush, unsigned end);
ZSTDMT_RustCompressStreamResult ZSTDMT_rust_compressStreamGeneric(
const ZSTDMT_RustCompressStreamContextProjection* context,
const ZSTDMT_RustStreamInputProjection* input,
const ZSTDMT_RustStreamOutputProjection* output,
unsigned end, void* opaque,
ZSTDMT_streamTryGetInputRangeFn tryGetInputRange,
ZSTDMT_streamLoadInputFn loadInput,
ZSTDMT_streamCreateJobFn createJob,
ZSTDMT_streamFlushProducedFn flushProduced);
typedef struct {
unsigned lastJob;
size_t srcSize;
@@ -1648,6 +1707,12 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
return result.returnCode;
}
static size_t ZSTDMT_streamCreateJob(void* opaque, size_t srcSize, unsigned end)
{
return ZSTDMT_createCompressionJob((ZSTDMT_CCtx*)opaque, srcSize,
(ZSTD_EndDirective)end);
}
/* The Rust flush state machine receives only this synchronized scalar view.
* The descriptor, condition variable, serial checksum state, and buffer pool
@@ -1763,6 +1828,16 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
return result.result;
}
static ZSTDMT_RustStreamFlushResult ZSTDMT_streamFlushProduced(
void* opaque, void* outputDst, size_t outputSize, size_t outputPos,
unsigned blockToFlush, unsigned end)
{
ZSTD_outBuffer output = { outputDst, outputSize, outputPos };
size_t const result = ZSTDMT_flushProduced(
(ZSTDMT_CCtx*)opaque, &output, blockToFlush, (ZSTD_EndDirective)end);
return (ZSTDMT_RustStreamFlushResult){ result, output.pos };
}
/**
* Returns the range of data used by the earliest job that is not yet complete.
* If the data of the first job is broken up into two segments, we cover both
@@ -1875,29 +1950,35 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
return 1;
}
typedef struct {
size_t toLoad; /* The number of bytes to load from the input. */
int flush; /* Boolean declaring if we must flush because we found a synchronization point. */
} SyncPoint;
/**
* Searches through the input for a synchronization point. If one is found, we
* will instruct the caller to flush, and return the number of bytes to load.
* Otherwise, we will load as many bytes as possible and instruct the caller
* to continue as normal.
*/
static SyncPoint
findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
/* Adapter callback for the C-owned reusable input range. */
static int ZSTDMT_streamTryGetInputRange(
void* opaque, ZSTDMT_RustStreamInputRangeProjection* projection)
{
SyncPoint syncPoint;
ZSTDMT_rust_findSynchronizationPoint(
input.src, input.size, input.pos,
mtctx->targetSectionSize,
mtctx->inBuff.buffer.start, mtctx->inBuff.filled,
mtctx->params.rsyncable,
mtctx->rsync.primePower, mtctx->rsync.hitMask,
&syncPoint.toLoad, &syncPoint.flush);
return syncPoint;
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
int const ready = ZSTDMT_tryGetInputRange(mtctx);
*projection = (ZSTDMT_RustStreamInputRangeProjection){
mtctx->inBuff.buffer.start,
mtctx->inBuff.buffer.capacity,
mtctx->inBuff.filled
};
return ready;
}
static int ZSTDMT_streamLoadInput(void* opaque, const void* src, size_t size)
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
assert(mtctx->inBuff.buffer.start != NULL);
assert(mtctx->inBuff.filled <= mtctx->inBuff.buffer.capacity);
assert(size <= mtctx->inBuff.buffer.capacity - mtctx->inBuff.filled);
ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, src, size);
mtctx->inBuff.filled += size;
return 1;
}
size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx)
@@ -1915,69 +1996,39 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp)
{
unsigned forwardInputProgress = 0;
ZSTDMT_RustCompressStreamContextProjection const context = {
mtctx->frameEnded,
(unsigned)mtctx->jobReady,
mtctx->inBuff.buffer.start,
mtctx->inBuff.buffer.capacity,
mtctx->inBuff.filled,
mtctx->targetSectionSize,
mtctx->params.rsyncable,
mtctx->rsync.primePower,
mtctx->rsync.hitMask
};
ZSTDMT_RustStreamInputProjection const inputProjection = {
input->src,
input->size,
input->pos
};
ZSTDMT_RustStreamOutputProjection const outputProjection = {
output->dst,
output->size,
output->pos
};
ZSTDMT_RustCompressStreamResult const result =
ZSTDMT_rust_compressStreamGeneric(
&context, &inputProjection, &outputProjection, (unsigned)endOp,
mtctx, ZSTDMT_streamTryGetInputRange, ZSTDMT_streamLoadInput,
ZSTDMT_streamCreateJob, ZSTDMT_streamFlushProduced);
DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)",
(U32)endOp, (U32)(input->size - input->pos));
assert(output->pos <= output->size);
assert(input->pos <= input->size);
if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
/* current frame being ended. Only flush/end are allowed */
return ERROR(stage_wrong);
}
/* fill input buffer */
if ( (!mtctx->jobReady)
&& (input->size > input->pos) ) { /* support NULL input */
if (mtctx->inBuff.buffer.start == NULL) {
assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */
if (!ZSTDMT_tryGetInputRange(mtctx)) {
/* It is only possible for this operation to fail if there are
* still compression jobs ongoing.
*/
DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed");
assert(mtctx->doneJobID != mtctx->nextJobID);
} else
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
}
if (mtctx->inBuff.buffer.start != NULL) {
SyncPoint const syncPoint = findSynchronizationPoint(mtctx, *input);
if (syncPoint.flush && endOp == ZSTD_e_continue) {
endOp = ZSTD_e_flush;
}
assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize);
DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u",
(U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad);
input->pos += syncPoint.toLoad;
mtctx->inBuff.filled += syncPoint.toLoad;
forwardInputProgress = syncPoint.toLoad>0;
}
}
if ((input->pos < input->size) && (endOp == ZSTD_e_end)) {
/* Can't end yet because the input is not fully consumed.
* We are in one of these cases:
* - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job.
* - We filled the input buffer: flush this job but don't end the frame.
* - We hit a synchronization point: flush this job but don't end the frame.
*/
assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable);
endOp = ZSTD_e_flush;
}
if ( (mtctx->jobReady)
|| (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */
|| ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */
|| ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */
size_t const jobSize = mtctx->inBuff.filled;
assert(mtctx->inBuff.filled <= mtctx->targetSectionSize);
FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , "");
}
/* check for potential compressed data ready to be flushed */
{ size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */
if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */
DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush);
return remainingToFlush;
}
input->pos = result.inputPos;
output->pos = result.outputPos;
DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)result.result);
return result.result;
}