ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move frame progression job aggregation into Rust

Browse Source 
Move the multithreaded frame-progression ring scan into Rust so the Rust
scheduler owns job-ID masking, ready-job range handling, error-output
normalization, and active-worker accounting. Keep C-owned job descriptors and
mutex-protected snapshots behind the narrow ZSTDMT_projectJob callback, which
preserves the private synchronization boundary while making the aggregation
policy directly testable in Rust.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (691 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && MAKEFLAGS=-j1 make -B -C programs -j1 zstd
- ulimit -v 41943040 && MAKEFLAGS=-j1 make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed)
This commit is contained in:
ddidderr
2026-07-19 18:53:08 +02:00
parent fec11ae7d0
commit adaae03552
3 changed files with 141 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+10 -21
View File
@@ -460,6 +460,11 @@ ZSTD_frameProgression ZSTDMT_rust_frameProgression(
ZSTD_frameProgression ZSTDMT_rust_frameProgressionAddJob(
ZSTD_frameProgression progression, size_t srcSize, size_t consumed,
size_t produced, size_t flushed);
ZSTD_frameProgression ZSTDMT_rust_frameProgressionWithJobs(
unsigned long long consumed, size_t inBuffFilled,
unsigned long long produced, unsigned currentJobID,
unsigned doneJobID, unsigned nextJobID, unsigned jobReady,
unsigned jobIDMask, void* opaque, ZSTDMT_jobProjectionFn projectJob);
typedef int (*ZSTDMT_jobTableInitFn)(void* jobTable, unsigned nbJobs,
size_t jobSize);
typedef void (*ZSTDMT_jobTableDestroyFn)(void* jobTable, unsigned nbJobs,
@@ -1523,28 +1528,12 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
* Note : mutex will be acquired during statistics collection inside workers. */
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
{
ZSTD_frameProgression fps = ZSTDMT_rust_frameProgression(
mtctx->consumed, mtctx->inBuff.filled, mtctx->produced,
mtctx->nextJobID);
ZSTD_frameProgression const fps = ZSTDMT_rust_frameProgressionWithJobs(
mtctx->consumed, mtctx->inBuff.filled, mtctx->produced,
mtctx->nextJobID, mtctx->doneJobID, mtctx->nextJobID,
mtctx->jobReady, mtctx->jobIDMask, mtctx,
ZSTDMT_projectJob);
DEBUGLOG(5, "ZSTDMT_getFrameProgression");
{ unsigned jobNb;
unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1);
DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)",
mtctx->doneJobID, lastJobNb, mtctx->jobReady);
for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) {
unsigned const wJobID = jobNb & mtctx->jobIDMask;
ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID];
ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
{ size_t const cResult = jobPtr->cSize;
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
assert(flushed <= produced);
fps = ZSTDMT_rust_frameProgressionAddJob(
fps, jobPtr->src.size, jobPtr->consumed, produced, flushed);
}
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
}
}
return fps;
}