feat(mt): move compression-job begin policy to Rust
The worker context previously selected the cdict or raw-prefix initialization path, applied non-first-job parameter updates, and published the frame-header projection directly in C. That left the branch and failure order intertwined with private CCtx and parameter layouts. Project only the first-job and cdict flags into Rust. Rust now validates cdict placement, selects the initialization path, stops on force-window or prefix-policy errors, and publishes the header projection only after successful initialization. C callbacks retain the private cdict, CCtx, parameter mutation, and frame-header field operations, including the original pledged-size and force-window behavior. Focused tests cover cdict ordering, non-first parameter ordering, parameter and initialization failures, header-publication suppression, and invalid cdict placement. Test Plan: - `rustfmt +nightly --edition 2021 --check rust/src/zstdmt_compress.rs` -- passed. - Capped GCC syntax-only check for `zstdmt_compress.c` -- passed. - Capped Clang syntax-only check for `zstdmt_compress.c` -- passed. - `git diff --cached --check` -- passed. - Cargo, native builds, fuzzers, and large tests were not run per assignment.
This commit is contained in:
@@ -222,6 +222,18 @@ typedef char ZSTDMT_compression_job_projection_layout[
|
||||
== 2 * sizeof(unsigned) + sizeof(void*))
|
||||
? 1 : -1];
|
||||
|
||||
typedef struct {
|
||||
unsigned firstJob;
|
||||
unsigned hasCDict;
|
||||
} ZSTDMT_RustCompressionJobBeginProjection;
|
||||
typedef char ZSTDMT_compression_job_begin_projection_layout[
|
||||
(offsetof(ZSTDMT_RustCompressionJobBeginProjection, firstJob) == 0
|
||||
&& offsetof(ZSTDMT_RustCompressionJobBeginProjection, hasCDict)
|
||||
== sizeof(unsigned)
|
||||
&& sizeof(ZSTDMT_RustCompressionJobBeginProjection)
|
||||
== 2 * sizeof(unsigned))
|
||||
? 1 : -1];
|
||||
|
||||
typedef struct {
|
||||
int status;
|
||||
size_t toFlush;
|
||||
@@ -235,6 +247,7 @@ typedef void (*ZSTDMT_compressionJobVoidFn)(void* opaque);
|
||||
typedef ZSTDMT_chunkProcessResult (*ZSTDMT_compressionJobCompressFn)(
|
||||
void* opaque, unsigned lastJob);
|
||||
typedef void (*ZSTDMT_compressionJobSizeFn)(void* opaque, size_t size);
|
||||
typedef size_t (*ZSTDMT_compressionJobSetParameterFn)(void* opaque, int value);
|
||||
|
||||
typedef struct {
|
||||
void* callbackContext;
|
||||
@@ -284,6 +297,15 @@ typedef char ZSTDMT_compression_job_parameters_layout[
|
||||
ZSTDMT_RustCompressionJobParameters ZSTDMT_rust_prepareCompressionJobParameters(
|
||||
unsigned jobID, int checksumFlag, int ldmEnable, unsigned nbWorkers);
|
||||
|
||||
size_t ZSTDMT_rust_compressionJobBegin(
|
||||
const ZSTDMT_RustCompressionJobBeginProjection* projection,
|
||||
void* opaque,
|
||||
ZSTDMT_compressionJobStepFn beginWithCDict,
|
||||
ZSTDMT_compressionJobSetParameterFn setForceMaxWindow,
|
||||
ZSTDMT_compressionJobStepFn setDeterministicRefPrefix,
|
||||
ZSTDMT_compressionJobStepFn beginWithPrefix,
|
||||
ZSTDMT_compressionJobVoidFn publishFrameHeader);
|
||||
|
||||
void ZSTDMT_rust_compressionJob(
|
||||
const ZSTDMT_RustCompressionJobProjection* projection,
|
||||
const ZSTDMT_RustCompressionJobFinishProjection* finishProjection,
|
||||
@@ -1702,36 +1724,79 @@ static void ZSTDMT_compressionJobGenerateSequences(void* opaque)
|
||||
ZSTDMT_serialState_advance);
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_compressionJobBeginWithCDict(void* opaque);
|
||||
static size_t ZSTDMT_compressionJobSetForceMaxWindow(void* opaque, int value);
|
||||
static size_t ZSTDMT_compressionJobSetDeterministicRefPrefix(void* opaque);
|
||||
static size_t ZSTDMT_compressionJobBeginWithPrefix(void* opaque);
|
||||
static void ZSTDMT_compressionJobPublishFrameHeader(void* opaque);
|
||||
|
||||
static size_t ZSTDMT_compressionJobBegin(void* opaque)
|
||||
{
|
||||
ZSTDMT_compressionJobState* const state =
|
||||
(ZSTDMT_compressionJobState*)opaque;
|
||||
ZSTDMT_jobDescription* const job = state->job;
|
||||
size_t initError;
|
||||
|
||||
if (job->cdict) {
|
||||
initError = ZSTD_compressBegin_advanced_internal(
|
||||
state->cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict,
|
||||
&state->jobParams, job->fullFrameSize);
|
||||
assert(job->firstJob); /* only allowed for first job */
|
||||
} else {
|
||||
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
|
||||
size_t const forceWindowError = ZSTD_CCtxParams_setParameter(
|
||||
&state->jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
|
||||
if (ZSTD_isError(forceWindowError)) return forceWindowError;
|
||||
if (!job->firstJob) {
|
||||
size_t const err = ZSTD_CCtxParams_setParameter(
|
||||
&state->jobParams, ZSTD_c_deterministicRefPrefix, 0);
|
||||
if (ZSTD_isError(err)) return err;
|
||||
}
|
||||
DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size);
|
||||
initError = ZSTD_compressBegin_advanced_internal(
|
||||
state->cctx, job->prefix.start, job->prefix.size,
|
||||
ZSTD_dct_rawContent, ZSTD_dtlm_fast, NULL, /*cdict*/
|
||||
&state->jobParams, pledgedSrcSize);
|
||||
}
|
||||
ZSTDMT_RustCompressionJobBeginProjection const projection = {
|
||||
job->firstJob,
|
||||
(unsigned)(job->cdict != NULL)
|
||||
};
|
||||
return ZSTDMT_rust_compressionJobBegin(
|
||||
&projection, state,
|
||||
ZSTDMT_compressionJobBeginWithCDict,
|
||||
ZSTDMT_compressionJobSetForceMaxWindow,
|
||||
ZSTDMT_compressionJobSetDeterministicRefPrefix,
|
||||
ZSTDMT_compressionJobBeginWithPrefix,
|
||||
ZSTDMT_compressionJobPublishFrameHeader);
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_compressionJobBeginWithCDict(void* opaque)
|
||||
{
|
||||
ZSTDMT_compressionJobState* const state =
|
||||
(ZSTDMT_compressionJobState*)opaque;
|
||||
ZSTDMT_jobDescription* const job = state->job;
|
||||
|
||||
return ZSTD_compressBegin_advanced_internal(
|
||||
state->cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict,
|
||||
&state->jobParams, job->fullFrameSize);
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_compressionJobSetForceMaxWindow(void* opaque, int value)
|
||||
{
|
||||
ZSTDMT_compressionJobState* const state =
|
||||
(ZSTDMT_compressionJobState*)opaque;
|
||||
|
||||
return ZSTD_CCtxParams_setParameter(
|
||||
&state->jobParams, ZSTD_c_forceMaxWindow, value);
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_compressionJobSetDeterministicRefPrefix(void* opaque)
|
||||
{
|
||||
ZSTDMT_compressionJobState* const state =
|
||||
(ZSTDMT_compressionJobState*)opaque;
|
||||
|
||||
return ZSTD_CCtxParams_setParameter(
|
||||
&state->jobParams, ZSTD_c_deterministicRefPrefix, 0);
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_compressionJobBeginWithPrefix(void* opaque)
|
||||
{
|
||||
ZSTDMT_compressionJobState* const state =
|
||||
(ZSTDMT_compressionJobState*)opaque;
|
||||
ZSTDMT_jobDescription* const job = state->job;
|
||||
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
|
||||
|
||||
DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size);
|
||||
return ZSTD_compressBegin_advanced_internal(
|
||||
state->cctx, job->prefix.start, job->prefix.size,
|
||||
ZSTD_dct_rawContent, ZSTD_dtlm_fast, NULL, /*cdict*/
|
||||
&state->jobParams, pledgedSrcSize);
|
||||
}
|
||||
|
||||
static void ZSTDMT_compressionJobPublishFrameHeader(void* opaque)
|
||||
{
|
||||
ZSTDMT_compressionJobState* const state =
|
||||
(ZSTDMT_compressionJobState*)opaque;
|
||||
|
||||
if (ZSTD_isError(initError)) return initError;
|
||||
state->frameHeaderState.stage = (int*)&state->cctx->stage;
|
||||
state->frameHeaderState.noDictIDFlag =
|
||||
state->cctx->appliedParams.fParams.noDictIDFlag;
|
||||
@@ -1747,7 +1812,6 @@ static size_t ZSTDMT_compressionJobBegin(void* opaque)
|
||||
state->frameHeaderState.dictID = state->cctx->dictID;
|
||||
state->frameHeaderState.repCodes = state->cctx->blockState.prevCBlock == NULL
|
||||
? NULL : state->cctx->blockState.prevCBlock->rep;
|
||||
return initError;
|
||||
}
|
||||
|
||||
static void ZSTDMT_compressionJobApplySequences(void* opaque)
|
||||
|
||||
Reference in New Issue
Block a user