ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move begin dictionary policy into Rust

Browse Source 
Move shared compression-begin dictionary selection, CDict attach
thresholds, reset ordering, insertion dispatch, and dictionary result
publication into a Rust-owned ABI leaf. Keep CCtx reset, CDict attach,
and private dictionary insertion behind C callbacks.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --lib
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- make -B -C programs -j1 zstd
- make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
- focused compress_begin unit tests
This commit is contained in:
ddidderr
2026-07-19 15:12:38 +02:00
parent cc8cc47da7
commit a3127511ce
2 changed files with 530 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+130 -30
View File
@@ -1488,6 +1488,73 @@ typedef char ZSTD_rust_init_cdict_state_layout[
&& sizeof(ZSTD_rust_initCDictState) == 19 * sizeof(void*))
? 1 : -1];
typedef size_t (*ZSTD_rust_compressBeginResetInternal_f)(
void* context, const void* params, U64 pledgedSrcSize,
size_t loadedDictSize, int zbuff);
typedef size_t (*ZSTD_rust_compressBeginResetUsingCDict_f)(
void* context, const void* cdict, const void* params,
U64 pledgedSrcSize, int zbuff);
typedef size_t (*ZSTD_rust_compressBeginInsertDictionary_f)(
void* context, const void* cdict, const void* dict,
size_t dictSize, int dictContentType, int dtlm);
typedef struct {
void* callbackContext;
const void* params;
const void* cdict;
const size_t* cdictContentSize;
const int* cdictCompressionLevel;
const ZSTD_dictAttachPref_e* attachDictPref;
const void* dict;
const size_t* dictSize;
const int* dictContentType;
const int* dtlm;
const U64* pledgedSrcSize;
const int* zbuff;
const int* forceLoad;
U32* dictID;
size_t* dictContentSize;
ZSTD_rust_compressBeginResetInternal_f resetInternal;
ZSTD_rust_compressBeginResetUsingCDict_f resetUsingCDict;
ZSTD_rust_compressBeginInsertDictionary_f insertDictionary;
} ZSTD_rust_compressBeginState;
size_t ZSTD_rust_compressBegin(const ZSTD_rust_compressBeginState* state);
typedef char ZSTD_rust_compress_begin_state_layout[
(offsetof(ZSTD_rust_compressBeginState, callbackContext) == 0
&& offsetof(ZSTD_rust_compressBeginState, params) == sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, cdict) == 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, cdictContentSize)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, cdictCompressionLevel)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, attachDictPref)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, dict)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, dictSize)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, dictContentType)
== 8 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, dtlm)
== 9 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, pledgedSrcSize)
== 10 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, zbuff)
== 11 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, forceLoad)
== 12 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, dictID)
== 13 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, dictContentSize)
== 14 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, resetInternal)
== 15 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, resetUsingCDict)
== 16 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginState, insertDictionary)
== 17 * sizeof(void*)
&& sizeof(ZSTD_rust_compressBeginState) == 18 * sizeof(void*))
? 1 : -1];
/* The sequence-compression loop receives only the state it actually reads or
* updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses
* the Rust ABI. */
@@ -4470,6 +4537,45 @@ static size_t ZSTD_rust_initCDict_insertDictionary(
ZSTD_tfp_forCDict, cdict->entropyWorkspace);
}
static size_t ZSTD_rust_compressBegin_resetInternal(
void* context, const void* params, U64 pledgedSrcSize,
size_t loadedDictSize, int zbuff)
{
return ZSTD_resetCCtx_internal(
(ZSTD_CCtx*)context, (const ZSTD_CCtx_params*)params,
pledgedSrcSize, loadedDictSize, ZSTDcrp_makeClean,
(ZSTD_buffered_policy_e)zbuff);
}
static size_t ZSTD_rust_compressBegin_resetUsingCDict(
void* context, const void* cdict, const void* params,
U64 pledgedSrcSize, int zbuff)
{
return ZSTD_resetCCtx_usingCDict(
(ZSTD_CCtx*)context, (const ZSTD_CDict*)cdict,
(const ZSTD_CCtx_params*)params, pledgedSrcSize,
(ZSTD_buffered_policy_e)zbuff);
}
static size_t ZSTD_rust_compressBegin_insertDictionary(
void* context, const void* cdict, const void* dict,
size_t dictSize, int dictContentType, int dtlm)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
if (cdict != NULL) {
ZSTD_CDict const* const dictionary = (const ZSTD_CDict*)cdict;
dict = dictionary->dictContent;
dictSize = dictionary->dictContentSize;
dictContentType = (int)dictionary->dictContentType;
}
return ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&cctx->ldmState, &cctx->workspace, &cctx->appliedParams,
dict, dictSize, (ZSTD_dictContentType_e)dictContentType,
(ZSTD_dictTableLoadMethod_e)dtlm, ZSTD_tfp_forCCtx,
cctx->tmpWorkspace);
}
#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
@@ -4484,7 +4590,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const ZSTD_CCtx_params* params, U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff)
{
size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize;
ZSTD_rust_compressBeginState state;
int const dictContentTypeValue = (int)dictContentType;
int const dtlmValue = (int)dtlm;
int const zbuffValue = (int)zbuff;
int const forceLoadValue = (int)ZSTD_dictForceLoad;
#if ZSTD_TRACE
cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
#endif
@@ -4492,35 +4602,25 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
if ( (cdict)
&& (cdict->dictContentSize > 0)
&& ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
|| pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
|| pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
|| cdict->compressionLevel == 0)
&& (params->attachDictPref != ZSTD_dictForceLoad) ) {
return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
}
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
dictContentSize,
ZSTDcrp_makeClean, zbuff) , "");
{ size_t const dictID = cdict ?
ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
cdict->dictContentSize, cdict->dictContentType, dtlm,
ZSTD_tfp_forCCtx, cctx->tmpWorkspace)
: ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace);
FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID;
cctx->dictContentSize = dictContentSize;
}
return 0;
state.callbackContext = cctx;
state.params = params;
state.cdict = cdict;
state.cdictContentSize = cdict == NULL ? NULL : &cdict->dictContentSize;
state.cdictCompressionLevel = cdict == NULL ? NULL : &cdict->compressionLevel;
state.attachDictPref = &params->attachDictPref;
state.dict = dict;
state.dictSize = &dictSize;
state.dictContentType = &dictContentTypeValue;
state.dtlm = &dtlmValue;
state.pledgedSrcSize = &pledgedSrcSize;
state.zbuff = &zbuffValue;
state.forceLoad = &forceLoadValue;
state.dictID = &cctx->dictID;
state.dictContentSize = &cctx->dictContentSize;
state.resetInternal = ZSTD_rust_compressBegin_resetInternal;
state.resetUsingCDict = ZSTD_rust_compressBegin_resetUsingCDict;
state.insertDictionary = ZSTD_rust_compressBegin_insertDictionary;
return ZSTD_rust_compressBegin(&state);
}
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,