feat(mt): move compression-job begin policy to Rust

The worker context previously selected the cdict or raw-prefix initialization
path, applied non-first-job parameter updates, and published the frame-header
projection directly in C. That left the branch and failure order intertwined
with private CCtx and parameter layouts.

Project only the first-job and cdict flags into Rust. Rust now validates cdict
placement, selects the initialization path, stops on force-window or
prefix-policy errors, and publishes the header projection only after successful
initialization. C callbacks retain the private cdict, CCtx, parameter mutation,
and frame-header field operations, including the original pledged-size and
force-window behavior.

Focused tests cover cdict ordering, non-first parameter ordering, parameter and
initialization failures, header-publication suppression, and invalid cdict
placement.

Test Plan:
- `rustfmt +nightly --edition 2021 --check rust/src/zstdmt_compress.rs` -- passed.
- Capped GCC syntax-only check for `zstdmt_compress.c` -- passed.
- Capped Clang syntax-only check for `zstdmt_compress.c` -- passed.
- `git diff --cached --check` -- passed.
- Cargo, native builds, fuzzers, and large tests were not run per assignment.
This commit is contained in:
2026-07-20 16:48:10 +02:00
parent 211659131d
commit 69a2d3969d
2 changed files with 326 additions and 24 deletions
+88 -24
View File
@@ -222,6 +222,18 @@ typedef char ZSTDMT_compression_job_projection_layout[
== 2 * sizeof(unsigned) + sizeof(void*))
? 1 : -1];
typedef struct {
unsigned firstJob;
unsigned hasCDict;
} ZSTDMT_RustCompressionJobBeginProjection;
typedef char ZSTDMT_compression_job_begin_projection_layout[
(offsetof(ZSTDMT_RustCompressionJobBeginProjection, firstJob) == 0
&& offsetof(ZSTDMT_RustCompressionJobBeginProjection, hasCDict)
== sizeof(unsigned)
&& sizeof(ZSTDMT_RustCompressionJobBeginProjection)
== 2 * sizeof(unsigned))
? 1 : -1];
typedef struct {
int status;
size_t toFlush;
@@ -235,6 +247,7 @@ typedef void (*ZSTDMT_compressionJobVoidFn)(void* opaque);
typedef ZSTDMT_chunkProcessResult (*ZSTDMT_compressionJobCompressFn)(
void* opaque, unsigned lastJob);
typedef void (*ZSTDMT_compressionJobSizeFn)(void* opaque, size_t size);
typedef size_t (*ZSTDMT_compressionJobSetParameterFn)(void* opaque, int value);
typedef struct {
void* callbackContext;
@@ -284,6 +297,15 @@ typedef char ZSTDMT_compression_job_parameters_layout[
ZSTDMT_RustCompressionJobParameters ZSTDMT_rust_prepareCompressionJobParameters(
unsigned jobID, int checksumFlag, int ldmEnable, unsigned nbWorkers);
size_t ZSTDMT_rust_compressionJobBegin(
const ZSTDMT_RustCompressionJobBeginProjection* projection,
void* opaque,
ZSTDMT_compressionJobStepFn beginWithCDict,
ZSTDMT_compressionJobSetParameterFn setForceMaxWindow,
ZSTDMT_compressionJobStepFn setDeterministicRefPrefix,
ZSTDMT_compressionJobStepFn beginWithPrefix,
ZSTDMT_compressionJobVoidFn publishFrameHeader);
void ZSTDMT_rust_compressionJob(
const ZSTDMT_RustCompressionJobProjection* projection,
const ZSTDMT_RustCompressionJobFinishProjection* finishProjection,
@@ -1702,36 +1724,79 @@ static void ZSTDMT_compressionJobGenerateSequences(void* opaque)
ZSTDMT_serialState_advance);
}
static size_t ZSTDMT_compressionJobBeginWithCDict(void* opaque);
static size_t ZSTDMT_compressionJobSetForceMaxWindow(void* opaque, int value);
static size_t ZSTDMT_compressionJobSetDeterministicRefPrefix(void* opaque);
static size_t ZSTDMT_compressionJobBeginWithPrefix(void* opaque);
static void ZSTDMT_compressionJobPublishFrameHeader(void* opaque);
static size_t ZSTDMT_compressionJobBegin(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
size_t initError;
if (job->cdict) {
initError = ZSTD_compressBegin_advanced_internal(
state->cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict,
&state->jobParams, job->fullFrameSize);
assert(job->firstJob); /* only allowed for first job */
} else {
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
size_t const forceWindowError = ZSTD_CCtxParams_setParameter(
&state->jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) return forceWindowError;
if (!job->firstJob) {
size_t const err = ZSTD_CCtxParams_setParameter(
&state->jobParams, ZSTD_c_deterministicRefPrefix, 0);
if (ZSTD_isError(err)) return err;
}
DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size);
initError = ZSTD_compressBegin_advanced_internal(
state->cctx, job->prefix.start, job->prefix.size,
ZSTD_dct_rawContent, ZSTD_dtlm_fast, NULL, /*cdict*/
&state->jobParams, pledgedSrcSize);
}
ZSTDMT_RustCompressionJobBeginProjection const projection = {
job->firstJob,
(unsigned)(job->cdict != NULL)
};
return ZSTDMT_rust_compressionJobBegin(
&projection, state,
ZSTDMT_compressionJobBeginWithCDict,
ZSTDMT_compressionJobSetForceMaxWindow,
ZSTDMT_compressionJobSetDeterministicRefPrefix,
ZSTDMT_compressionJobBeginWithPrefix,
ZSTDMT_compressionJobPublishFrameHeader);
}
static size_t ZSTDMT_compressionJobBeginWithCDict(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
return ZSTD_compressBegin_advanced_internal(
state->cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict,
&state->jobParams, job->fullFrameSize);
}
static size_t ZSTDMT_compressionJobSetForceMaxWindow(void* opaque, int value)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
return ZSTD_CCtxParams_setParameter(
&state->jobParams, ZSTD_c_forceMaxWindow, value);
}
static size_t ZSTDMT_compressionJobSetDeterministicRefPrefix(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
return ZSTD_CCtxParams_setParameter(
&state->jobParams, ZSTD_c_deterministicRefPrefix, 0);
}
static size_t ZSTDMT_compressionJobBeginWithPrefix(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size);
return ZSTD_compressBegin_advanced_internal(
state->cctx, job->prefix.start, job->prefix.size,
ZSTD_dct_rawContent, ZSTD_dtlm_fast, NULL, /*cdict*/
&state->jobParams, pledgedSrcSize);
}
static void ZSTDMT_compressionJobPublishFrameHeader(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
if (ZSTD_isError(initError)) return initError;
state->frameHeaderState.stage = (int*)&state->cctx->stage;
state->frameHeaderState.noDictIDFlag =
state->cctx->appliedParams.fParams.noDictIDFlag;
@@ -1747,7 +1812,6 @@ static size_t ZSTDMT_compressionJobBegin(void* opaque)
state->frameHeaderState.dictID = state->cctx->dictID;
state->frameHeaderState.repCodes = state->cctx->blockState.prevCBlock == NULL
? NULL : state->cctx->blockState.prevCBlock->rep;
return initError;
}
static void ZSTDMT_compressionJobApplySequences(void* opaque)