ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move compression chunk loop into Rust

Browse Source 
ZSTDMT_compressionJob kept the chunking kernel in C while Rust already owned
most of the MT pool and progress helpers. The loop mixed compression calls
with mutex and condition-variable publication, making it difficult to test
its exact final-chunk and partial-tail behavior independently.

Move only the chunk-processing kernel into Rust. It calls the existing C
compression entry points through an explicit ABI, reports the final block size
or error in a repr(C) result, and invokes a C progress callback after each
successful intermediate chunk. C retains worker setup, serial and raw-sequence
state, synchronization, error publication, cleanup, and final completion
reporting. Mock compressors cover empty, exact, partial, progress, and error
boundaries without mirroring private MT job state in Rust.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --lib zstdmt_compress --
  --test-threads=1` -- passed (44 tests).
- `cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings` -- passed.
- `cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check` -- passed.
- `ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream` -- passed,
  including multi-threaded dictionary/checksum coverage.
This commit is contained in:
ddidderr
2026-07-18 18:46:13 +02:00
parent 1b85c430b1
commit 22bebab928
2 changed files with 368 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+32 -29
View File
@@ -116,6 +116,18 @@ void ZSTDMT_rust_buffer_pool_release(ZSTDMT_RustBufferPool* pool,
ZSTDMT_RustBuffer ZSTDMT_rust_buffer_pool_resize(ZSTDMT_RustBufferPool* pool,
ZSTDMT_RustBuffer buffer);
typedef struct {
size_t error;
size_t lastBlockSize;
} ZSTDMT_chunkProcessResult;
typedef void (*ZSTDMT_chunkProgressFn)(void* opaque, size_t cSize, size_t consumed);
ZSTDMT_chunkProcessResult ZSTDMT_rust_compressJobChunks(
ZSTD_CCtx* cctx, const void* src, size_t srcSize,
void* dst, size_t dstCapacity, size_t chunkSize, unsigned lastJob,
void* progressContext, ZSTDMT_chunkProgressFn progressCallback);
typedef struct {
rawSeq* seq;
size_t pos;
@@ -660,6 +672,18 @@ typedef struct {
unsigned frameChecksumNeeded; /* used only by mtctx */
} ZSTDMT_jobDescription;
static void ZSTDMT_compressionJobProgress(void* opaque, size_t cSize, size_t consumed)
{
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)opaque;
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
job->cSize += cSize;
job->consumed = consumed;
DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)",
(U32)cSize, (U32)job->cSize);
ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */
ZSTD_pthread_mutex_unlock(&job->job_mutex);
}
#define JOB_ERROR(e) \
do { \
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \
@@ -739,40 +763,19 @@ static void ZSTDMT_compressionJob(void* jobDescription)
/* compress the entire job by smaller chunks, for better granularity */
{ size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX;
int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize);
const BYTE* ip = (const BYTE*) job->src.start;
BYTE* const ostart = (BYTE*)dstBuff.start;
BYTE* op = ostart;
BYTE* oend = op + dstBuff.capacity;
int chunkNb;
if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */
DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks);
assert(job->cSize == 0);
for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) {
size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize);
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
ip += chunkSize;
op += cSize; assert(op < oend);
/* stats */
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
job->cSize += cSize;
job->consumed = chunkSize * chunkNb;
DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)",
(U32)cSize, (U32)job->cSize);
ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */
ZSTD_pthread_mutex_unlock(&job->job_mutex);
}
/* last block */
assert(chunkSize > 0);
assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */
if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) {
size_t const lastBlockSize1 = job->src.size & (chunkSize-1);
size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1;
size_t const cSize = (job->lastJob) ?
ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) :
ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize);
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
lastCBlockSize = cSize;
} }
{ ZSTDMT_chunkProcessResult const result = ZSTDMT_rust_compressJobChunks(
cctx, job->src.start, job->src.size,
dstBuff.start, dstBuff.capacity, chunkSize, job->lastJob,
job, ZSTDMT_compressionJobProgress);
if (ZSTD_isError(result.error)) JOB_ERROR(result.error);
lastCBlockSize = result.lastBlockSize;
}
}
if (!job->firstJob) {
/* Double check that we don't have an ext-dict, because then our
* repcode invalidation doesn't work.