feat(mt): move output publication into Rust
Move the bounded multithreaded output-publication kernel into Rust. Rust now calculates the flush amount, copies the selected job bytes, and returns the updated output and job offsets through an explicit C-layout result. C retains mutex and condition-variable handling, checksum insertion, job retirement, progress accounting, and the terminal state machine. The Rust helper validates the scalar bounds before copying and has focused coverage for empty output space, offset partial flushes, repeated flushes, complete flushes, sentinels, and invalid bounds. Test Plan: - cargo test --manifest-path rust/Cargo.toml --lib zstdmt_compress -- --test-threads=1 - cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check (blocked by concurrent fileio worker formatting its uncommitted file) - ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream (worker) - git diff --cached --check
This commit is contained in:
@@ -121,6 +121,13 @@ typedef struct {
|
||||
size_t lastBlockSize;
|
||||
} ZSTDMT_chunkProcessResult;
|
||||
|
||||
typedef struct {
|
||||
int status;
|
||||
size_t toFlush;
|
||||
size_t outputPos;
|
||||
size_t dstFlushed;
|
||||
} ZSTDMT_flushPublicationResult;
|
||||
|
||||
typedef void (*ZSTDMT_chunkProgressFn)(void* opaque, size_t cSize, size_t consumed);
|
||||
|
||||
ZSTDMT_chunkProcessResult ZSTDMT_rust_compressJobChunks(
|
||||
@@ -128,6 +135,11 @@ ZSTDMT_chunkProcessResult ZSTDMT_rust_compressJobChunks(
|
||||
void* dst, size_t dstCapacity, size_t chunkSize, unsigned lastJob,
|
||||
void* progressContext, ZSTDMT_chunkProgressFn progressCallback);
|
||||
|
||||
ZSTDMT_flushPublicationResult ZSTDMT_rust_publishJobOutput(
|
||||
void* outputDst, size_t outputSize, size_t outputPos,
|
||||
const void* jobDst, size_t jobCapacity,
|
||||
size_t cSize, size_t dstFlushed);
|
||||
|
||||
typedef struct {
|
||||
rawSeq* seq;
|
||||
size_t pos;
|
||||
@@ -1495,19 +1507,22 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
|
||||
}
|
||||
|
||||
if (cSize > 0) { /* compression is ongoing or completed */
|
||||
size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos);
|
||||
ZSTDMT_flushPublicationResult const publication = ZSTDMT_rust_publishJobOutput(
|
||||
output->dst, output->size, output->pos,
|
||||
mtctx->jobs[wJobID].dstBuff.start, mtctx->jobs[wJobID].dstBuff.capacity,
|
||||
cSize, mtctx->jobs[wJobID].dstFlushed);
|
||||
if (publication.status != 0) {
|
||||
assert(publication.status == 0);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)",
|
||||
(U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize);
|
||||
(U32)publication.toFlush, mtctx->doneJobID,
|
||||
(U32)srcConsumed, (U32)srcSize, (U32)cSize);
|
||||
assert(mtctx->doneJobID < mtctx->nextJobID);
|
||||
assert(cSize >= mtctx->jobs[wJobID].dstFlushed);
|
||||
assert(mtctx->jobs[wJobID].dstBuff.start != NULL);
|
||||
if (toFlush > 0) {
|
||||
ZSTD_memcpy((char*)output->dst + output->pos,
|
||||
(const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
|
||||
toFlush);
|
||||
}
|
||||
output->pos += toFlush;
|
||||
mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
|
||||
output->pos = publication.outputPos;
|
||||
mtctx->jobs[wJobID].dstFlushed = publication.dstFlushed; /* can write : this value is only used by mtctx */
|
||||
|
||||
if ( (srcConsumed == srcSize) /* job is completed */
|
||||
&& (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */
|
||||
|
||||
Reference in New Issue
Block a user