feat(compress): move MT job-table storage lifecycle to Rust

The multithreaded compressor already uses Rust-owned buffer and CCtx pools,
but zstdmt_compress.c still allocated and freed its job table directly. Move
the raw table storage and power-of-two sizing behind Rust's custom allocator
ABI. Keep descriptor field access and platform mutex/condition initialization
in C because those layouts remain private and platform-specific. The C
wrapper initializes and destroys synchronization primitives around the Rust
storage calls, preserving failure cleanup while leaving worker job setup,
scheduling, and stream entry points in C as the fallback implementation.

Keep the Rust storage-only ABI free of MT-only C references so single-threaded
archives can omit zstdmt_compress.c without acquiring new unresolved symbols.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --no-default-features
  --features compression zstdmt_compress --lib` -- passed (5 tests).
- `make -B -C lib lib-mt` -- passed.
- `make -B -C tests -j2 fullbench zstreamtest poolTests` -- passed.
- `./poolTests` -- passed.
- `./fullbench -i1 -B1000 README.md` -- passed, including -T2 scenarios.
- Rebuilt `programs/zstd` and ran a `-T2` compress/decompress `cmp` round-trip
  -- passed.
- `rustfmt` and `cargo +nightly fmt --manifest-path rust/Cargo.toml --all
  -- --check` -- passed.
- Full `cargo clippy -D warnings` remains blocked by unrelated warnings in
  the concurrent `rust/src/fileio_asyncio.rs` worktree changes.
- `./zstreamtest -T5s` reaches an unrelated single-thread maxBlockSize
  assertion at `tests/zstreamtest.c:2157`; MT fullbench and CLI smoke pass.
This commit is contained in:
2026-07-18 01:40:18 +02:00
parent 5c448120cf
commit a15bf1d375
3 changed files with 129 additions and 22 deletions
+34 -17
View File
@@ -796,13 +796,39 @@ struct ZSTDMT_CCtx_s {
static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem)
{
U32 jobNb;
if (jobTable == NULL) return;
ZSTDMT_job_table_destroy_sync(jobTable, nbJobs, sizeof(*jobTable));
ZSTDMT_rust_job_table_free(jobTable, nbJobs, sizeof(*jobTable), cMem);
}
int ZSTDMT_job_table_init_sync(void* jobTable, unsigned nbJobs, size_t jobSize)
{
U32 jobNb;
int initError = 0;
BYTE* const table = (BYTE*)jobTable;
assert(jobSize == sizeof(ZSTDMT_jobDescription));
if (jobTable == NULL) return 1;
for (jobNb=0; jobNb<nbJobs; jobNb++) {
ZSTD_pthread_mutex_destroy(&jobTable[jobNb].job_mutex);
ZSTD_pthread_cond_destroy(&jobTable[jobNb].job_cond);
ZSTDMT_jobDescription* const job =
(ZSTDMT_jobDescription*)(table + jobNb * jobSize);
initError |= ZSTD_pthread_mutex_init(&job->job_mutex, NULL);
initError |= ZSTD_pthread_cond_init(&job->job_cond, NULL);
}
return initError;
}
void ZSTDMT_job_table_destroy_sync(void* jobTable, unsigned nbJobs, size_t jobSize)
{
U32 jobNb;
BYTE* const table = (BYTE*)jobTable;
if (jobTable == NULL) return;
assert(jobSize == sizeof(ZSTDMT_jobDescription));
for (jobNb=0; jobNb<nbJobs; jobNb++) {
ZSTDMT_jobDescription* const job =
(ZSTDMT_jobDescription*)(table + jobNb * jobSize);
ZSTD_pthread_mutex_destroy(&job->job_mutex);
ZSTD_pthread_cond_destroy(&job->job_cond);
}
ZSTD_customFree(jobTable, cMem);
}
/* ZSTDMT_allocJobsTable()
@@ -810,20 +836,11 @@ static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZS
* update *nbJobsPtr to next power of 2 value, as size of table */
static ZSTDMT_jobDescription* ZSTDMT_createJobsTable(U32* nbJobsPtr, ZSTD_customMem cMem)
{
U32 const nbJobsLog2 = ZSTD_highbit32(*nbJobsPtr) + 1;
U32 const nbJobs = 1 << nbJobsLog2;
U32 jobNb;
ZSTDMT_jobDescription* const jobTable = (ZSTDMT_jobDescription*)
ZSTD_customCalloc(nbJobs * sizeof(ZSTDMT_jobDescription), cMem);
int initError = 0;
if (jobTable==NULL) return NULL;
*nbJobsPtr = nbJobs;
for (jobNb=0; jobNb<nbJobs; jobNb++) {
initError |= ZSTD_pthread_mutex_init(&jobTable[jobNb].job_mutex, NULL);
initError |= ZSTD_pthread_cond_init(&jobTable[jobNb].job_cond, NULL);
}
if (initError != 0) {
ZSTDMT_freeJobsTable(jobTable, nbJobs, cMem);
ZSTDMT_rust_job_table_create(nbJobsPtr, sizeof(ZSTDMT_jobDescription), cMem);
if (jobTable == NULL) return NULL;
if (ZSTDMT_job_table_init_sync(jobTable, *nbJobsPtr, sizeof(*jobTable)) != 0) {
ZSTDMT_freeJobsTable(jobTable, *nbJobsPtr, cMem);
return NULL;
}
return jobTable;
+11
View File
@@ -43,6 +43,17 @@
/* === Memory management === */
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
/* The Rust adapter owns job-table storage and custom allocator cleanup. The
* descriptor layout and platform synchronization primitives remain in C, so
* these callbacks are the explicit C fallback seam for job initialization. */
void* ZSTDMT_rust_job_table_create(unsigned* nbJobsPtr, size_t jobSize,
ZSTD_customMem cMem);
void ZSTDMT_rust_job_table_free(void* jobTable, unsigned nbJobs,
size_t jobSize, ZSTD_customMem cMem);
int ZSTDMT_job_table_init_sync(void* jobTable, unsigned nbJobs, size_t jobSize);
void ZSTDMT_job_table_destroy_sync(void* jobTable, unsigned nbJobs, size_t jobSize);
/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
ZSTD_customMem cMem,