ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move advanced CDict policy into Rust

Browse Source 
ZSTD_createCDict_advanced2 previously selected compression parameters,
resolved dedicated-dictionary-search fallback, and ordered allocation and
initialization entirely in C. That left a large public dictionary boundary
outside the Rust rewrite and made the failure ordering implicit in the C
wrapper.

The Rust params API now owns advanced-CDict parameter preparation, including
the dedicated-search override/fallback and row-matchfinder resolution. A Rust
dictionary bridge owns the create -> init sequence and frees a created CDict
when initialization fails. The C shim retains only the private workspace and
custom-memory allocation, dictionary initialization, and teardown callbacks;
opaque parameter-field pointers and compile-time layout assertions preserve
the existing ABI. Focused probes cover policy publication and all callback
failure-ordering paths, and the migration README records the remaining C
boundary.

Test Plan:
- `rustfmt --check` and `cargo fmt --manifest-path rust/Cargo.toml -- --check` -- passed
- `cargo test --manifest-path rust/Cargo.toml --release` -- 663 passed
- `cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings` -- passed
- `make -B -C programs -j1 zstd` -- passed
- `make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s` -- 84 tests and both fuzz rounds passed
This commit is contained in:
ddidderr
2026-07-19 16:53:31 +02:00
parent 35ad752b00
commit 5449a4b3c0
4 changed files with 684 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+97 -52
View File
@@ -1195,8 +1195,6 @@ size_t ZSTD_rust_params_maxNbSeq(size_t blockSize, U32 minMatch,
int useSequenceProducer);
size_t ZSTD_rust_params_resolveMaxBlockSize(size_t maxBlockSize);
size_t ZSTD_rust_params_getBlockSize(size_t maxBlockSize, U32 windowLog);
void ZSTD_rust_params_overrideCParams(ZSTD_compressionParameters* cParams,
const ZSTD_compressionParameters* overrides);
int ZSTD_rust_params_resolveExternalSequenceValidation(int mode);
int ZSTD_rust_params_rowMatchFinderSupported(int strategy);
int ZSTD_rust_params_rowMatchFinderUsed(int strategy, int mode);
@@ -1210,9 +1208,6 @@ int ZSTD_rust_params_resolveEnableLdm(
int mode, ZSTD_compressionParameters cParams);
int ZSTD_rust_params_resolveExternalRepcodeSearch(int mode, int cLevel);
int ZSTD_rust_params_cdictIndicesAreTagged(ZSTD_compressionParameters cParams);
int ZSTD_rust_params_dedicatedDictSearchIsSupported(ZSTD_compressionParameters cParams);
ZSTD_compressionParameters
ZSTD_rust_params_dedicatedDictSearch_getCParams(ZSTD_compressionParameters cParams);
ZSTD_compressionParameters
ZSTD_rust_params_dedicatedDictSearch_revertCParams(ZSTD_compressionParameters cParams);
int ZSTD_rust_params_getCParamMode(int cdict_present, int cdict_strategy,
@@ -1784,6 +1779,56 @@ typedef char ZSTD_rust_init_cdict_state_layout[
&& sizeof(ZSTD_rust_initCDictState) == 19 * sizeof(void*))
? 1 : -1];
typedef void* (*ZSTD_rust_createCDictAdvancedCreate_f)(
void* context, size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch);
typedef size_t (*ZSTD_rust_createCDictAdvancedInit_f)(
void* context, void* cdict, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
const ZSTD_CCtx_params* cctxParams);
typedef void (*ZSTD_rust_createCDictAdvancedFree_f)(
void* context, void* cdict);
typedef struct {
void* callbackContext;
ZSTD_CCtx_params* cctxParams;
const ZSTD_compressionParameters* cParams;
const int* enableDedicatedDictSearch;
const int* useRowMatchFinder;
U32 exclusionMask;
U32 ldmDefaultWindowLog;
ZSTD_rust_createCDictAdvancedCreate_f create;
ZSTD_rust_createCDictAdvancedInit_f init;
ZSTD_rust_createCDictAdvancedFree_f free;
} ZSTD_rust_createCDictAdvancedState;
void* ZSTD_rust_createCDictAdvanced(
const ZSTD_rust_createCDictAdvancedState* state,
const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType);
typedef char ZSTD_rust_create_cdict_advanced_state_layout[
(offsetof(ZSTD_rust_createCDictAdvancedState, callbackContext) == 0
&& offsetof(ZSTD_rust_createCDictAdvancedState, cctxParams)
== sizeof(void*)
&& offsetof(ZSTD_rust_createCDictAdvancedState, cParams)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_createCDictAdvancedState, enableDedicatedDictSearch)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_createCDictAdvancedState, useRowMatchFinder)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_createCDictAdvancedState, exclusionMask)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_createCDictAdvancedState, ldmDefaultWindowLog)
== 5 * sizeof(void*) + sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, create)
== 5 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, init)
== 6 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, free)
== 7 * sizeof(void*) + 2 * sizeof(U32)
&& sizeof(ZSTD_rust_createCDictAdvancedState)
== 8 * sizeof(void*) + 2 * sizeof(U32))
? 1 : -1];
typedef size_t (*ZSTD_rust_compressBeginResetInternal_f)(
void* context, const void* params, U64 pledgedSrcSize,
size_t loadedDictSize, int zbuff);
@@ -2707,9 +2752,6 @@ size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSr
&cctx->pledgedSrcSizePlusOne);
}
static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(
int const compressionLevel,
size_t const dictSize);
static void ZSTD_dedicatedDictSearch_revertCParams(
ZSTD_compressionParameters* cParams);
@@ -5558,6 +5600,36 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
&cctxParams, customMem);
}
static void* ZSTD_rust_createCDictAdvanced_create(
void* context, size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch)
{
ZSTD_customMem const customMem = *(const ZSTD_customMem*)context;
return ZSTD_createCDict_advanced_internal(
dictSize, (ZSTD_dictLoadMethod_e)dictLoadMethod, *cParams,
(ZSTD_ParamSwitch_e)useRowMatchFinder,
enableDedicatedDictSearch, customMem);
}
static size_t ZSTD_rust_createCDictAdvanced_init(
void* context, void* cdict, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
const ZSTD_CCtx_params* cctxParams)
{
(void)context;
return ZSTD_initCDict_internal(
(ZSTD_CDict*)cdict, dict, dictSize,
(ZSTD_dictLoadMethod_e)dictLoadMethod,
(ZSTD_dictContentType_e)dictContentType, *cctxParams);
}
static void ZSTD_rust_createCDictAdvanced_free(void* context, void* cdict)
{
(void)context;
ZSTD_freeCDict((ZSTD_CDict*)cdict);
}
ZSTD_CDict* ZSTD_createCDict_advanced2(
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
@@ -5565,47 +5637,27 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
const ZSTD_CCtx_params* originalCctxParams,
ZSTD_customMem customMem)
{
ZSTD_CCtx_params cctxParams = *originalCctxParams;
ZSTD_compressionParameters cParams;
ZSTD_CDict* cdict;
ZSTD_CCtx_params cctxParams;
ZSTD_rust_createCDictAdvancedState state;
DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
if (originalCctxParams == NULL) return NULL;
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
if (cctxParams.enableDedicatedDictSearch) {
cParams = ZSTD_dedicatedDictSearch_getCParams(
cctxParams.compressionLevel, dictSize);
ZSTD_rust_params_overrideCParams(&cParams, &cctxParams.cParams);
} else {
cParams = ZSTD_getCParamsFromCCtxParams(
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
}
if (!ZSTD_rust_params_dedicatedDictSearchIsSupported(cParams)) {
/* Fall back to non-DDSS params */
cctxParams.enableDedicatedDictSearch = 0;
cParams = ZSTD_getCParamsFromCCtxParams(
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
}
DEBUGLOG(3, "ZSTD_createCDict_advanced2: DedicatedDictSearch=%u", cctxParams.enableDedicatedDictSearch);
cctxParams.cParams = cParams;
cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
cdict = ZSTD_createCDict_advanced_internal(dictSize,
dictLoadMethod, cctxParams.cParams,
cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch,
customMem);
if (!cdict || ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize,
dictLoadMethod, dictContentType,
cctxParams) )) {
ZSTD_freeCDict(cdict);
return NULL;
}
return cdict;
cctxParams = *originalCctxParams;
state.callbackContext = &customMem;
state.cctxParams = &cctxParams;
state.cParams = &cctxParams.cParams;
state.enableDedicatedDictSearch = &cctxParams.enableDedicatedDictSearch;
state.useRowMatchFinder = (const int*)&cctxParams.useRowMatchFinder;
state.exclusionMask = ZSTD_getCParamsExclusionMask();
state.ldmDefaultWindowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
state.create = ZSTD_rust_createCDictAdvanced_create;
state.init = ZSTD_rust_createCDictAdvanced_init;
state.free = ZSTD_rust_createCDictAdvanced_free;
return (ZSTD_CDict*)ZSTD_rust_createCDictAdvanced(
&state, dict, dictSize,
(int)dictLoadMethod, (int)dictContentType);
}
static void* ZSTD_rust_createCDict_create(
@@ -7072,13 +7124,6 @@ int ZSTD_maxCLevel(void) { return ZSTD_rust_params_maxCLevel(); }
int ZSTD_minCLevel(void) { return ZSTD_rust_params_minCLevel(); }
int ZSTD_defaultCLevel(void) { return ZSTD_rust_params_defaultCLevel(); }
static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize)
{
ZSTD_compressionParameters const cParams =
ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict);
return ZSTD_rust_params_dedicatedDictSearch_getCParams(cParams);
}
/**
* Reverses the adjustment applied to cparams when enabling dedicated dict
* search. This is used to recover the params set to be used in the working