ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move job-ring decisions into Rust

Browse Source 
Move multithreaded compression's input-retention and pending-output policy
behind scalar job projections. Rust scans the ring and selects source or
prefix ranges while C keeps job descriptors, mutexes, pointer ownership, and
worker-private layout behind the projection callback.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --all-targets -- --test-threads=1
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- make -B -C lib -j2 lib
- make -B -C programs -j2 zstd
- make -B -C tests -j2 test-fuzzer
- make -B -C tests -j2 test-zstd
This commit is contained in:
ddidderr
2026-07-18 22:08:46 +02:00
parent f3bc5e98f1
commit 380d8075c6
2 changed files with 307 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+49 -57
View File
@@ -177,6 +177,28 @@ void ZSTDMT_rust_findSynchronizationPoint(const void* inputSrc, size_t inputSize
U64 ZSTDMT_rust_rollingHashPrimePower(U32 length);
size_t ZSTDMT_rust_nextInputSizeHint(size_t targetSectionSize,
size_t inBuffFilled);
typedef struct {
size_t consumed;
size_t cSize;
const void* srcStart;
size_t srcSize;
const void* prefixStart;
size_t prefixSize;
size_t dstFlushed;
} ZSTDMT_RustJobProjection;
typedef struct {
const void* start;
size_t size;
} ZSTDMT_RustInputRange;
typedef void (*ZSTDMT_jobProjectionFn)(void* opaque, unsigned jobID,
ZSTDMT_RustJobProjection* projection);
ZSTDMT_RustInputRange ZSTDMT_rust_getInputDataInUse(
unsigned firstJobID, unsigned lastJobID, unsigned jobIDMask,
size_t roundBufferCapacity, size_t targetSectionSize,
void* opaque, ZSTDMT_jobProjectionFn projectJob);
size_t ZSTDMT_rust_toFlushNow(unsigned doneJobID, unsigned nextJobID,
unsigned jobIDMask, void* opaque,
ZSTDMT_jobProjectionFn projectJob);
size_t ZSTDMT_rust_sizeofCCtx(size_t mtctxSize, size_t factorySize,
size_t bufferPoolSize, size_t jobsSize,
size_t cctxPoolSize, size_t seqPoolSize,
@@ -885,6 +907,24 @@ struct ZSTDMT_CCtx_s {
unsigned providedFactory: 1;
};
/* Project only the scalar job state needed by Rust's read-only MT
* orchestration. The descriptor layout and its mutex remain private here. */
static void ZSTDMT_projectJob(void* opaque, unsigned jobID,
ZSTDMT_RustJobProjection* projection)
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
ZSTDMT_jobDescription* const job = &mtctx->jobs[jobID];
ZSTD_pthread_mutex_lock(&job->job_mutex);
projection->consumed = job->consumed;
projection->cSize = job->cSize;
projection->dstFlushed = job->dstFlushed;
ZSTD_pthread_mutex_unlock(&job->job_mutex);
projection->srcStart = job->src.start;
projection->srcSize = job->src.size;
projection->prefixStart = job->prefix.start;
projection->prefixSize = job->prefix.size;
}
static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem)
{
if (jobTable == NULL) return;
@@ -1160,34 +1200,10 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
{
size_t toFlush;
unsigned const jobID = mtctx->doneJobID;
assert(jobID <= mtctx->nextJobID);
if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */
/* look into oldest non-fully-flushed job */
{ unsigned const wJobID = jobID & mtctx->jobIDMask;
ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID];
ZSTD_pthread_mutex_lock(&jobPtr->job_mutex);
{ size_t const cResult = jobPtr->cSize;
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
assert(flushed <= produced);
assert(jobPtr->consumed <= jobPtr->src.size);
toFlush = produced - flushed;
/* if toFlush==0, nothing is available to flush.
* However, jobID is expected to still be active:
* if jobID was already completed and fully flushed,
* ZSTDMT_flushProduced() should have already moved onto next job.
* Therefore, some input has not yet been consumed. */
if (toFlush==0) {
assert(jobPtr->consumed < jobPtr->src.size);
}
}
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
}
return toFlush;
assert(mtctx->doneJobID <= mtctx->nextJobID);
return ZSTDMT_rust_toFlushNow(mtctx->doneJobID, mtctx->nextJobID,
mtctx->jobIDMask, mtctx,
ZSTDMT_projectJob);
}
@@ -1556,35 +1572,11 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
*/
static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
{
unsigned const firstJobID = mtctx->doneJobID;
unsigned const lastJobID = mtctx->nextJobID;
unsigned jobID;
/* no need to check during first round */
size_t roundBuffCapacity = mtctx->roundBuff.capacity;
size_t nbJobs1stRoundMin = roundBuffCapacity / mtctx->targetSectionSize;
if (lastJobID < nbJobs1stRoundMin) return kNullRange;
for (jobID = firstJobID; jobID < lastJobID; ++jobID) {
unsigned const wJobID = jobID & mtctx->jobIDMask;
size_t consumed;
ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex);
consumed = mtctx->jobs[wJobID].consumed;
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
if (consumed < mtctx->jobs[wJobID].src.size) {
Range range = mtctx->jobs[wJobID].prefix;
if (range.size == 0) {
/* Empty prefix */
range = mtctx->jobs[wJobID].src;
}
/* Job source in multiple segments not supported yet */
assert(range.start <= mtctx->jobs[wJobID].src.start);
return range;
}
}
return kNullRange;
ZSTDMT_RustInputRange const range = ZSTDMT_rust_getInputDataInUse(
mtctx->doneJobID, mtctx->nextJobID, mtctx->jobIDMask,
mtctx->roundBuff.capacity, mtctx->targetSectionSize,
mtctx, ZSTDMT_projectJob);
return (Range){ range.start, range.size };
}
/**