ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CDict begin policy into Rust

Browse Source 
Move ZSTD_compressBegin_usingCDict_internal source-size parameter selection,
parameter initialization ordering, and source-window floor into a Rust-owned
ABI bridge while retaining private CCtx/CDict layouts and the final C begin
callback. Preserve strict cutoff, dictionary-size multiplier, unknown-size,
level-zero, and begin-error semantics.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --lib
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- make -B -C programs -j1 zstd
- make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
- focused cdict_begin unit tests
This commit is contained in:
ddidderr
2026-07-19 14:19:43 +02:00
parent 148af6d884
commit 4952b259c4
2 changed files with 535 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+94 -29
View File
@@ -1226,6 +1226,57 @@ size_t ZSTD_rust_CCtx_refPrefixAdvanced(
const void* prefix, size_t prefixSize, int dictContentType,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignPrefixDict_f assignPrefixDict);
typedef void (*ZSTD_rust_compressBeginUsingCDictInitParams_f)(
void* cctxParams, const ZSTD_parameters* params, int compressionLevel);
typedef void (*ZSTD_rust_compressBeginUsingCDictAdjustWindow_f)(
void* cctxParams, U32 minWindowLog);
typedef size_t (*ZSTD_rust_compressBeginUsingCDictBegin_f)(
void* context, const void* cdict, const void* cctxParams,
U64 pledgedSrcSize);
typedef struct {
void* cctx;
const void* cdict;
void* cctxParams;
const ZSTD_compressionParameters* cdictCParams;
const size_t* cdictContentSize;
const int* cdictCompressionLevel;
const ZSTD_frameParameters* fParams;
const unsigned long long* pledgedSrcSize;
const U32* exclusionMask;
ZSTD_rust_compressBeginUsingCDictInitParams_f initParams;
ZSTD_rust_compressBeginUsingCDictAdjustWindow_f adjustWindow;
ZSTD_rust_compressBeginUsingCDictBegin_f begin;
} ZSTD_rust_compressBeginUsingCDictState;
size_t ZSTD_rust_compressBeginUsingCDict(
const ZSTD_rust_compressBeginUsingCDictState* state);
typedef char ZSTD_rust_compress_begin_using_cdict_state_layout[
(offsetof(ZSTD_rust_compressBeginUsingCDictState, cctx) == 0
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, cdict)
== sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, cctxParams)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, cdictCParams)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, cdictContentSize)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState,
cdictCompressionLevel)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, fParams)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, pledgedSrcSize)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, exclusionMask)
== 8 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, initParams)
== 9 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, adjustWindow)
== 10 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingCDictState, begin)
== 11 * sizeof(void*)
&& sizeof(ZSTD_rust_compressBeginUsingCDictState)
== 12 * sizeof(void*))
? 1 : -1];
size_t ZSTD_rust_transferSequencesWBlockDelim(
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
@@ -4902,6 +4953,34 @@ unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict)
return cdict->dictID;
}
static void ZSTD_rust_compressBeginUsingCDict_initParams(
void* cctxParams, const ZSTD_parameters* params, int compressionLevel)
{
ZSTD_CCtxParams_init_internal(
(ZSTD_CCtx_params*)cctxParams, params, compressionLevel);
}
static void ZSTD_rust_compressBeginUsingCDict_adjustWindow(
void* cctxParams, U32 minWindowLog)
{
ZSTD_CCtx_params* const params = (ZSTD_CCtx_params*)cctxParams;
if (params->cParams.windowLog < minWindowLog) {
params->cParams.windowLog = minWindowLog;
}
}
static size_t ZSTD_rust_compressBeginUsingCDict_begin(
void* context, const void* cdict, const void* cctxParams,
U64 pledgedSrcSize)
{
return ZSTD_compressBegin_internal(
(ZSTD_CCtx*)context, NULL, 0,
ZSTD_dct_auto, ZSTD_dtlm_fast,
(const ZSTD_CDict*)cdict,
(const ZSTD_CCtx_params*)cctxParams,
pledgedSrcSize, ZSTDb_not_buffered);
}
/* ZSTD_compressBegin_usingCDict_internal() :
* Implementation of various ZSTD_compressBegin_usingCDict* functions.
*/
@@ -4910,36 +4989,22 @@ static size_t ZSTD_compressBegin_usingCDict_internal(
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
{
ZSTD_CCtx_params cctxParams;
U32 const exclusionMask = ZSTD_getCParamsExclusionMask();
ZSTD_rust_compressBeginUsingCDictState state;
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal");
RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
/* Initialize the cctxParams from the cdict */
{
ZSTD_parameters params;
params.fParams = fParams;
params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
|| pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
|| pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
|| cdict->compressionLevel == 0 ) ?
ZSTD_getCParamsFromCDict(cdict)
: ZSTD_getCParams(cdict->compressionLevel,
pledgedSrcSize,
cdict->dictContentSize);
ZSTD_CCtxParams_init_internal(&cctxParams, &params, cdict->compressionLevel);
}
/* Increase window log to fit the entire dictionary and source if the
* source size is known. Limit the increase to 19, which is the
* window log for compression level 1 with the largest source size.
*/
if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog);
}
return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast,
cdict,
&cctxParams, pledgedSrcSize,
ZSTDb_not_buffered);
state.cctx = cctx;
state.cdict = cdict;
state.cctxParams = &cctxParams;
state.cdictCParams = cdict == NULL ? NULL : &cdict->matchState.cParams;
state.cdictContentSize = cdict == NULL ? NULL : &cdict->dictContentSize;
state.cdictCompressionLevel = cdict == NULL ? NULL : &cdict->compressionLevel;
state.fParams = &fParams;
state.pledgedSrcSize = &pledgedSrcSize;
state.exclusionMask = &exclusionMask;
state.initParams = ZSTD_rust_compressBeginUsingCDict_initParams;
state.adjustWindow = ZSTD_rust_compressBeginUsingCDict_adjustWindow;
state.begin = ZSTD_rust_compressBeginUsingCDict_begin;
return ZSTD_rust_compressBeginUsingCDict(&state);
}