ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): aggregate MT frame progression in Rust

Browse Source 
Move the arithmetic aggregation in ZSTDMT_getFrameProgression across the
existing C/Rust boundary. C still owns MT state traversal, mutex locking,
jobIDMask ordering, nextJobID plus jobReady handling, error normalization,
and the flushed <= produced assertion. Rust constructs the base result and
folds each compact, C-normalized job snapshot into the complete repr(C)
ZSTD_frameProgression, preserving size_t-to-U64 wrapping and active-worker
counting without exposing private job or mutex layouts.

Focused Rust tests cover zero jobs, a ready job, completed jobs, normalized
error output, and mixed active/completed aggregation.

Test Plan:
- All three compression clippy commands passed before formatting.
- cargo +nightly fmt --manifest-path rust/Cargo.toml -- passed.
- All three compression clippy commands passed after formatting.
- cargo test --manifest-path rust/Cargo.toml --no-default-features
  --features compression -- 332 passed.
- make -B -C lib -j2 lib-mt -- passed.
- make -B -C lib -j2 lib-nomt -- passed.
- make -B -C tests -j2 test-zstream -- passed: 84 named, 6,004, and 8,392
  randomized cases.
- git diff --check and git diff --cached --check -- passed.

The zstream build retained the pre-existing warning at
tests/zstreamtest.c:1899 about an unterminated initializer string.
This commit is contained in:
ddidderr
2026-07-18 14:05:46 +02:00
parent 4c114b8e7b
commit 3bd4dba2ee
2 changed files with 143 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+11 -11
View File
@@ -157,6 +157,12 @@ size_t ZSTDMT_rust_sizeofCCtx(size_t mtctxSize, size_t factorySize,
size_t bufferPoolSize, size_t jobsSize,
size_t cctxPoolSize, size_t seqPoolSize,
size_t cdictSize, size_t roundBuffSize);
ZSTD_frameProgression ZSTDMT_rust_frameProgression(
unsigned long long consumed, size_t inBuffFilled,
unsigned long long produced, unsigned currentJobID);
ZSTD_frameProgression ZSTDMT_rust_frameProgressionAddJob(
ZSTD_frameProgression progression, size_t srcSize, size_t consumed,
size_t produced, size_t flushed);
typedef struct ZSTDMT_bufferPool_s {
ZSTDMT_RustBufferPool* rustPool;
@@ -1101,13 +1107,10 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
* Note : mutex will be acquired during statistics collection inside workers. */
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
{
ZSTD_frameProgression fps;
ZSTD_frameProgression fps = ZSTDMT_rust_frameProgression(
mtctx->consumed, mtctx->inBuff.filled, mtctx->produced,
mtctx->nextJobID);
DEBUGLOG(5, "ZSTDMT_getFrameProgression");
fps.ingested = mtctx->consumed + mtctx->inBuff.filled;
fps.consumed = mtctx->consumed;
fps.produced = fps.flushed = mtctx->produced;
fps.currentJobID = mtctx->nextJobID;
fps.nbActiveWorkers = 0;
{ unsigned jobNb;
unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1);
DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)",
@@ -1120,11 +1123,8 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
assert(flushed <= produced);
fps.ingested += jobPtr->src.size;
fps.consumed += jobPtr->consumed;
fps.produced += produced;
fps.flushed += flushed;
fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size);
fps = ZSTDMT_rust_frameProgressionAddJob(
fps, jobPtr->src.size, jobPtr->consumed, produced, flushed);
}
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
}