ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CDict initialization policy into Rust

Browse Source 
Move shared CDict initialization ordering and scalar publication into a
Rust-owned ABI leaf used by both dynamic and static CDict creation. Keep
content allocation/copy, entropy workspace reservation, match-state reset,
and dictionary insertion behind narrow C callbacks so private CDict and
workspace layouts remain C-owned.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --lib
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- make -B -C programs -j1 zstd
- make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
- focused cdict_init unit test
This commit is contained in:
ddidderr
2026-07-19 15:02:02 +02:00
parent 73ba17a76d
commit 7775396fae
2 changed files with 502 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+148 -42
View File
@@ -1414,6 +1414,80 @@ typedef char ZSTD_rust_external_sequence_store_state_layout[
== 5 * sizeof(void*))
? 1 : -1];
typedef size_t (*ZSTD_rust_initCDictAssignContent_f)(
void* context, const void* dict, size_t dictSize, int dictLoadMethod);
typedef void* (*ZSTD_rust_initCDictReserveEntropy_f)(void* context);
typedef void (*ZSTD_rust_initCDictResetBlockState_f)(void* context);
typedef size_t (*ZSTD_rust_initCDictResetMatchState_f)(
void* context, const ZSTD_compressionParameters* cParams,
int useRowMatchFinder);
typedef size_t (*ZSTD_rust_initCDictInsertDictionary_f)(
void* context, const void* params, const void* dict,
size_t dictSize, int dictContentType);
typedef struct {
void* callbackContext;
void* params;
const ZSTD_compressionParameters* cParams;
ZSTD_compressionParameters* matchStateCParams;
int* dedicatedDictSearch;
const int* enableDedicatedDictSearch;
const ZSTD_ParamSwitch_e* useRowMatchFinder;
const void** dictContent;
size_t* dictContentSize;
int* dictContentType;
U32** entropyWorkspace;
U32* dictID;
int* compressionLevel;
int* contentSizeFlag;
ZSTD_rust_initCDictAssignContent_f assignContent;
ZSTD_rust_initCDictReserveEntropy_f reserveEntropy;
ZSTD_rust_initCDictResetBlockState_f resetBlockState;
ZSTD_rust_initCDictResetMatchState_f resetMatchState;
ZSTD_rust_initCDictInsertDictionary_f insertDictionary;
} ZSTD_rust_initCDictState;
size_t ZSTD_rust_initCDict(
const ZSTD_rust_initCDictState* state,
const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType);
typedef char ZSTD_rust_init_cdict_state_layout[
(offsetof(ZSTD_rust_initCDictState, callbackContext) == 0
&& offsetof(ZSTD_rust_initCDictState, params) == sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, cParams) == 2 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, matchStateCParams)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, dedicatedDictSearch)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, enableDedicatedDictSearch)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, useRowMatchFinder)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, dictContent)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, dictContentSize)
== 8 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, dictContentType)
== 9 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, entropyWorkspace)
== 10 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, dictID)
== 11 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, compressionLevel)
== 12 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, contentSizeFlag)
== 13 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, assignContent)
== 14 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, reserveEntropy)
== 15 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, resetBlockState)
== 16 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, resetMatchState)
== 17 * sizeof(void*)
&& offsetof(ZSTD_rust_initCDictState, insertDictionary)
== 18 * sizeof(void*)
&& sizeof(ZSTD_rust_initCDictState) == 19 * sizeof(void*))
? 1 : -1];
/* The sequence-compression loop receives only the state it actually reads or
* updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses
* the Rust ABI. */
@@ -4345,6 +4419,57 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
ZSTD_loadDictionaryContent_callback);
}
static size_t ZSTD_rust_initCDict_assignContent(
void* context, const void* dict, size_t dictSize, int dictLoadMethod)
{
ZSTD_CDict* const cdict = (ZSTD_CDict*)context;
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
cdict->dictContent = dict;
} else {
void* const internalBuffer = ZSTD_cwksp_reserve_object(
&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*)));
RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!");
cdict->dictContent = internalBuffer;
ZSTD_memcpy(internalBuffer, dict, dictSize);
}
return 0;
}
static void* ZSTD_rust_initCDict_reserveEntropy(void* context)
{
ZSTD_CDict* const cdict = (ZSTD_CDict*)context;
return ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE);
}
static void ZSTD_rust_initCDict_resetBlockState(void* context)
{
ZSTD_CDict* const cdict = (ZSTD_CDict*)context;
ZSTD_reset_compressedBlockState(&cdict->cBlockState);
}
static size_t ZSTD_rust_initCDict_resetMatchState(
void* context, const ZSTD_compressionParameters* cParams,
int useRowMatchFinder)
{
ZSTD_CDict* const cdict = (ZSTD_CDict*)context;
return ZSTD_reset_matchState(
&cdict->matchState, &cdict->workspace, cParams,
(ZSTD_ParamSwitch_e)useRowMatchFinder,
ZSTDcrp_makeClean, ZSTDirp_reset, ZSTD_resetTarget_CDict);
}
static size_t ZSTD_rust_initCDict_insertDictionary(
void* context, const void* params, const void* dict,
size_t dictSize, int dictContentType)
{
ZSTD_CDict* const cdict = (ZSTD_CDict*)context;
return ZSTD_compress_insertDictionary(
&cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
(const ZSTD_CCtx_params*)params, dict, dictSize,
(ZSTD_dictContentType_e)dictContentType, ZSTD_dtlm_full,
ZSTD_tfp_forCDict, cdict->entropyWorkspace);
}
#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
@@ -4763,50 +4888,31 @@ static size_t ZSTD_initCDict_internal(
ZSTD_dictContentType_e dictContentType,
ZSTD_CCtx_params params)
{
ZSTD_rust_initCDictState state;
DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType);
assert(!ZSTD_checkCParams(params.cParams));
cdict->matchState.cParams = params.cParams;
cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch;
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
cdict->dictContent = dictBuffer;
} else {
void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*)));
RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!");
cdict->dictContent = internalBuffer;
ZSTD_memcpy(internalBuffer, dictBuffer, dictSize);
}
cdict->dictContentSize = dictSize;
cdict->dictContentType = dictContentType;
cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE);
/* Reset the state to no dictionary */
ZSTD_reset_compressedBlockState(&cdict->cBlockState);
FORWARD_IF_ERROR(ZSTD_reset_matchState(
&cdict->matchState,
&cdict->workspace,
&params.cParams,
params.useRowMatchFinder,
ZSTDcrp_makeClean,
ZSTDirp_reset,
ZSTD_resetTarget_CDict), "");
/* (Maybe) load the dictionary
* Skips loading the dictionary if it is < 8 bytes.
*/
{ params.compressionLevel = ZSTD_CLEVEL_DEFAULT;
params.fParams.contentSizeFlag = 1;
{ size_t const dictID = ZSTD_compress_insertDictionary(
&cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
&params, cdict->dictContent, cdict->dictContentSize,
dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace);
FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
assert(dictID <= (size_t)(U32)-1);
cdict->dictID = (U32)dictID;
}
}
return 0;
state.callbackContext = cdict;
state.params = &params;
state.cParams = &params.cParams;
state.matchStateCParams = &cdict->matchState.cParams;
state.dedicatedDictSearch = &cdict->matchState.dedicatedDictSearch;
state.enableDedicatedDictSearch = &params.enableDedicatedDictSearch;
state.useRowMatchFinder = &params.useRowMatchFinder;
state.dictContent = &cdict->dictContent;
state.dictContentSize = &cdict->dictContentSize;
state.dictContentType = (int*)&cdict->dictContentType;
state.entropyWorkspace = &cdict->entropyWorkspace;
state.dictID = &cdict->dictID;
state.compressionLevel = &params.compressionLevel;
state.contentSizeFlag = &params.fParams.contentSizeFlag;
state.assignContent = ZSTD_rust_initCDict_assignContent;
state.reserveEntropy = ZSTD_rust_initCDict_reserveEntropy;
state.resetBlockState = ZSTD_rust_initCDict_resetBlockState;
state.resetMatchState = ZSTD_rust_initCDict_resetMatchState;
state.insertDictionary = ZSTD_rust_initCDict_insertDictionary;
return ZSTD_rust_initCDict(
&state, dictBuffer, dictSize,
(int)dictLoadMethod, (int)dictContentType);
}
static ZSTD_CDict*