ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cdict): move static workspace sizing policy to Rust

Browse Source 
ZSTD_initStaticCDict previously resolved automatic row matching and computed the
minimum workspace in C before handing the result to the Rust construction
orchestrator. That left the public static-CDict sizing branch split from the
Rust validation and callback failure policy.

Make the Rust projection carry dictionary size, load method, compression
parameters, and the C-owned layout sizing inputs. Rust now resolves automatic
row matching, computes the dedicated-search workspace requirement, validates
workspace capacity, and passes the resolved mode into the initializer while
preserving create/reserve/move/init order. C retains opaque private cwksp and
CDict operations, allocator/layout behavior, and dictionary-content loading.

Add focused tests for computed sizing policy, capacity boundaries, callback
order, reservation and initialization failures, and null or unaligned
workspaces.

Test Plan:
- `rustfmt --edition 2021 --check rust/src/zstd_compress_dictionary.rs` -- passed
- GCC `-fsyntax-only` checks with multithreaded, non-multithreaded,
  `ZSTD_DISABLE_ASM=1`, and `ZSTD_ADDRESS_SANITIZER=1` configurations -- passed
- Clang `-fsyntax-only` check with `ZSTD_MULTITHREAD` -- passed
- `git diff --check` and `git diff --cached --check` -- passed
- Cargo, make, native builds, and large tests were not run per task constraints
This commit is contained in:
ddidderr
2026-07-20 16:25:52 +02:00
parent 1499677913
commit cb2c1aede1
2 changed files with 164 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+28 -22
View File
@@ -462,14 +462,17 @@ typedef void* (*ZSTD_rust_initStaticCDictReserveObject_f)(void* context);
typedef void (*ZSTD_rust_initStaticCDictMoveWorkspace_f)(
void* context, void* cdict);
typedef void (*ZSTD_rust_initStaticCDictInitialize_f)(
void* context, void* cdict);
void* context, void* cdict, int useRowMatchFinder);
typedef void* (*ZSTD_rust_initStaticCDictInit_f)(
void* context, void* cdict);
typedef struct {
void* callbackContext;
void* workspace;
size_t workspaceSize;
size_t neededSize;
size_t dictSize;
size_t dictLoadMethod;
const ZSTD_compressionParameters* cParams;
const void* sizing;
ZSTD_rust_initStaticCDictCreateWorkspace_f createWorkspace;
ZSTD_rust_initStaticCDictReserveObject_f reserveObject;
ZSTD_rust_initStaticCDictMoveWorkspace_f moveWorkspace;
@@ -483,19 +486,25 @@ typedef char ZSTD_rust_init_static_cdict_state_layout[
== sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, workspaceSize)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, neededSize)
&& offsetof(ZSTD_rust_initStaticCDictState, dictSize)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, createWorkspace)
&& offsetof(ZSTD_rust_initStaticCDictState, dictLoadMethod)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, reserveObject)
&& offsetof(ZSTD_rust_initStaticCDictState, cParams)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, moveWorkspace)
&& offsetof(ZSTD_rust_initStaticCDictState, sizing)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, initialize)
&& offsetof(ZSTD_rust_initStaticCDictState, createWorkspace)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, init)
&& offsetof(ZSTD_rust_initStaticCDictState, reserveObject)
== 8 * sizeof(void*)
&& sizeof(ZSTD_rust_initStaticCDictState) == 9 * sizeof(void*))
&& offsetof(ZSTD_rust_initStaticCDictState, moveWorkspace)
== 9 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, initialize)
== 10 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, init)
== 11 * sizeof(void*)
&& sizeof(ZSTD_rust_initStaticCDictState) == 12 * sizeof(void*))
? 1 : -1];
typedef void (*ZSTD_rust_clearAllDictsCallback_f)(void* context);
typedef struct {
@@ -7252,7 +7261,6 @@ typedef struct {
ZSTD_dictLoadMethod_e dictLoadMethod;
ZSTD_dictContentType_e dictContentType;
ZSTD_compressionParameters cParams;
ZSTD_ParamSwitch_e useRowMatchFinder;
ZSTD_cwksp workspaceState;
ZSTD_CCtx_params params;
} ZSTD_rust_initStaticCDictContext;
@@ -7280,7 +7288,8 @@ static void ZSTD_rust_initStaticCDict_moveWorkspace(
ZSTD_cwksp_move(&((ZSTD_CDict*)cdict)->workspace, &context->workspaceState);
}
static void ZSTD_rust_initStaticCDict_initialize(void* opaque, void* cdict)
static void ZSTD_rust_initStaticCDict_initialize(
void* opaque, void* cdict, int useRowMatchFinder)
{
ZSTD_rust_initStaticCDictContext* const context =
(ZSTD_rust_initStaticCDictContext*)opaque;
@@ -7289,8 +7298,8 @@ static void ZSTD_rust_initStaticCDict_initialize(void* opaque, void* cdict)
ZSTD_CCtxParams_init(params, 0);
params->cParams = context->cParams;
params->useRowMatchFinder = context->useRowMatchFinder;
dictionary->useRowMatchFinder = context->useRowMatchFinder;
params->useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
dictionary->useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
dictionary->compressionLevel = ZSTD_NO_CLEVEL;
}
@@ -7313,8 +7322,6 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams)
{
ZSTD_ParamSwitch_e const useRowMatchFinder = (ZSTD_ParamSwitch_e)
ZSTD_resolveRowMatchFinderMode((int)ZSTD_ps_auto, cParams);
ZSTD_rustCDictSizing const sizing = {
sizeof(ZSTD_CDict),
HUF_WORKSPACE_SIZE,
@@ -7323,15 +7330,12 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
sizeof(ZSTD_optimal_t),
ZSTD_RUST_ASAN_REDZONE_SIZE
};
/* Dedicated search keeps the match state large enough for a later DDS
* plus row-hash use of this static CDict. */
size_t const neededSize = ZSTD_rust_params_estimateCDictWorkspaceSize(
dictSize, cParams, (int)dictLoadMethod,
useRowMatchFinder, 1, &sizing);
ZSTD_rust_initStaticCDictContext context;
ZSTD_rust_initStaticCDictState state;
DEBUGLOG(4, "ZSTD_initStaticCDict (dictSize==%u)", (unsigned)dictSize);
/* Rust resolves row matching and computes the required workspace size;
* these callbacks retain only private workspace and CDict operations. */
context.workspace = workspace;
context.workspaceSize = workspaceSize;
context.dict = dict;
@@ -7339,11 +7343,13 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
context.dictLoadMethod = dictLoadMethod;
context.dictContentType = dictContentType;
context.cParams = cParams;
context.useRowMatchFinder = useRowMatchFinder;
state.callbackContext = &context;
state.workspace = workspace;
state.workspaceSize = workspaceSize;
state.neededSize = neededSize;
state.dictSize = dictSize;
state.dictLoadMethod = (size_t)dictLoadMethod;
state.cParams = &cParams;
state.sizing = &sizing;
state.createWorkspace = ZSTD_rust_initStaticCDict_createWorkspace;
state.reserveObject = ZSTD_rust_initStaticCDict_reserveObject;
state.moveWorkspace = ZSTD_rust_initStaticCDict_moveWorkspace;