ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move job resource acquisition policy to Rust

Browse Source 
MT worker execution used to combine CCtx acquisition, raw-sequence-store
acquisition, destination-buffer allocation, frame-header publication, and the
LDM resource check in one C callback. That made the resource order and
short-circuit policy another C-owned implementation boundary even though the
operations themselves must remain private to C.

Project the scalar LDM and destination-presence inputs into Rust and let Rust
own the ordering and error normalization. The C callbacks now only acquire
private resources, report readiness, allocate the destination buffer, and
publish the frame-header destination. The ordering remains compatible with the
worker cleanup path: both pool gets happen first, CCtx failure stops before
destination work, destination failure stops before publication, and the LDM
sequence-store check runs after publication.

Test Plan:
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` -- passed
- `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check` -- passed
- Capped GCC and Clang syntax-only checks for `lib/compress/zstdmt_compress.c` -- passed by the worker
- `git diff --cached --check` -- passed
- Native build and full upstream tests are deferred to the post-batch serial verification.
This commit is contained in:
ddidderr
2026-07-20 17:20:12 +02:00
parent 62a9db5f65
commit 97469a2e55
2 changed files with 344 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+83 -14
View File
@@ -294,9 +294,33 @@ typedef char ZSTDMT_compression_job_parameters_layout[
== 2 * sizeof(int) + sizeof(unsigned))
? 1 : -1];
typedef struct {
int ldmEnabled;
int hasDestination;
} ZSTDMT_RustCompressionJobResourceProjection;
typedef char ZSTDMT_compression_job_resource_projection_layout[
(offsetof(ZSTDMT_RustCompressionJobResourceProjection, ldmEnabled) == 0
&& offsetof(ZSTDMT_RustCompressionJobResourceProjection, hasDestination)
== sizeof(int)
&& sizeof(ZSTDMT_RustCompressionJobResourceProjection)
== 2 * sizeof(int))
? 1 : -1];
typedef int (*ZSTDMT_compressionJobResourceReadyFn)(void* opaque);
ZSTDMT_RustCompressionJobParameters ZSTDMT_rust_prepareCompressionJobParameters(
unsigned jobID, int checksumFlag, int ldmEnable, unsigned nbWorkers);
size_t ZSTDMT_rust_compressionJobAcquireResources(
const ZSTDMT_RustCompressionJobResourceProjection* projection,
void* opaque,
ZSTDMT_compressionJobVoidFn acquireCCtx,
ZSTDMT_compressionJobVoidFn acquireRawSeqStore,
ZSTDMT_compressionJobResourceReadyFn hasCCtx,
ZSTDMT_compressionJobStepFn acquireDestinationBuffer,
ZSTDMT_compressionJobResourceReadyFn hasRawSeqStore,
ZSTDMT_compressionJobVoidFn publishDestination);
size_t ZSTDMT_rust_compressionJobBegin(
const ZSTDMT_RustCompressionJobBeginProjection* projection,
void* opaque,
@@ -1664,28 +1688,73 @@ typedef struct {
ZSTDMT_RustCompressionJobFrameHeaderState frameHeaderState;
} ZSTDMT_compressionJobState;
static void ZSTDMT_compressionJobAcquireCCtx(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
state->cctx = ZSTDMT_getCCtx(state->job->cctxPool);
}
static void ZSTDMT_compressionJobAcquireRawSeqStore(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
state->rawSeqStore = ZSTDMT_getSeq(state->job->seqPool);
}
static int ZSTDMT_compressionJobHasCCtx(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
return state->cctx != NULL;
}
static size_t ZSTDMT_compressionJobAcquireDestinationBuffer(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
state->dstBuff = ZSTDMT_getBuffer(job->bufPool);
if (state->dstBuff.start == NULL) return ERROR(memory_allocation);
job->dstBuff = state->dstBuff;
return 0;
}
static int ZSTDMT_compressionJobHasRawSeqStore(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
return state->rawSeqStore.seq != NULL;
}
static void ZSTDMT_compressionJobPublishDestination(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
state->frameHeaderState.dst = state->dstBuff.start;
state->frameHeaderState.dstCapacity = state->dstBuff.capacity;
}
static size_t ZSTDMT_compressionJobAcquireResources(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
state->cctx = ZSTDMT_getCCtx(job->cctxPool);
state->rawSeqStore = ZSTDMT_getSeq(job->seqPool);
state->dstBuff = job->dstBuff;
DEBUGLOG(5, "ZSTDMT_compressionJob: job %u", job->jobID);
if (state->cctx == NULL) return ERROR(memory_allocation);
if (state->dstBuff.start == NULL) {
state->dstBuff = ZSTDMT_getBuffer(job->bufPool);
if (state->dstBuff.start == NULL) return ERROR(memory_allocation);
job->dstBuff = state->dstBuff;
}
state->frameHeaderState.dst = state->dstBuff.start;
state->frameHeaderState.dstCapacity = state->dstBuff.capacity;
if (state->jobParams.ldmParams.enableLdm == ZSTD_ps_enable &&
state->rawSeqStore.seq == NULL)
return ERROR(memory_allocation);
return 0;
ZSTDMT_RustCompressionJobResourceProjection const projection = {
state->jobParams.ldmParams.enableLdm,
state->dstBuff.start != NULL
};
return ZSTDMT_rust_compressionJobAcquireResources(
&projection, state,
ZSTDMT_compressionJobAcquireCCtx,
ZSTDMT_compressionJobAcquireRawSeqStore,
ZSTDMT_compressionJobHasCCtx,
ZSTDMT_compressionJobAcquireDestinationBuffer,
ZSTDMT_compressionJobHasRawSeqStore,
ZSTDMT_compressionJobPublishDestination);
}
static void ZSTDMT_compressionJobPrepareParameters(void* opaque)