feat(mt): move output publication into Rust

Move the bounded multithreaded output-publication kernel into Rust.  Rust now
calculates the flush amount, copies the selected job bytes, and returns the
updated output and job offsets through an explicit C-layout result.  C retains
mutex and condition-variable handling, checksum insertion, job retirement,
progress accounting, and the terminal state machine.

The Rust helper validates the scalar bounds before copying and has focused
coverage for empty output space, offset partial flushes, repeated flushes,
complete flushes, sentinels, and invalid bounds.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --lib zstdmt_compress -- --test-threads=1
- cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings
- cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check (blocked by concurrent fileio worker formatting its uncommitted file)
- ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream (worker)
- git diff --cached --check
This commit is contained in:
2026-07-18 19:00:22 +02:00
parent 428dfbc4f5
commit fb6d6d3219
2 changed files with 252 additions and 9 deletions
+24 -9
View File
@@ -121,6 +121,13 @@ typedef struct {
size_t lastBlockSize;
} ZSTDMT_chunkProcessResult;
typedef struct {
int status;
size_t toFlush;
size_t outputPos;
size_t dstFlushed;
} ZSTDMT_flushPublicationResult;
typedef void (*ZSTDMT_chunkProgressFn)(void* opaque, size_t cSize, size_t consumed);
ZSTDMT_chunkProcessResult ZSTDMT_rust_compressJobChunks(
@@ -128,6 +135,11 @@ ZSTDMT_chunkProcessResult ZSTDMT_rust_compressJobChunks(
void* dst, size_t dstCapacity, size_t chunkSize, unsigned lastJob,
void* progressContext, ZSTDMT_chunkProgressFn progressCallback);
ZSTDMT_flushPublicationResult ZSTDMT_rust_publishJobOutput(
void* outputDst, size_t outputSize, size_t outputPos,
const void* jobDst, size_t jobCapacity,
size_t cSize, size_t dstFlushed);
typedef struct {
rawSeq* seq;
size_t pos;
@@ -1495,19 +1507,22 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
}
if (cSize > 0) { /* compression is ongoing or completed */
size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos);
ZSTDMT_flushPublicationResult const publication = ZSTDMT_rust_publishJobOutput(
output->dst, output->size, output->pos,
mtctx->jobs[wJobID].dstBuff.start, mtctx->jobs[wJobID].dstBuff.capacity,
cSize, mtctx->jobs[wJobID].dstFlushed);
if (publication.status != 0) {
assert(publication.status == 0);
return ERROR(GENERIC);
}
DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)",
(U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize);
(U32)publication.toFlush, mtctx->doneJobID,
(U32)srcConsumed, (U32)srcSize, (U32)cSize);
assert(mtctx->doneJobID < mtctx->nextJobID);
assert(cSize >= mtctx->jobs[wJobID].dstFlushed);
assert(mtctx->jobs[wJobID].dstBuff.start != NULL);
if (toFlush > 0) {
ZSTD_memcpy((char*)output->dst + output->pos,
(const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
toFlush);
}
output->pos += toFlush;
mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
output->pos = publication.outputPos;
mtctx->jobs[wJobID].dstFlushed = publication.dstFlushed; /* can write : this value is only used by mtctx */
if ( (srcConsumed == srcSize) /* job is completed */
&& (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */