ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move advanced CDict allocation policy into Rust

Browse Source 
The advanced CDict bridge previously left custom-memory validation,
workspace allocation, and create/init failure ordering inside one C helper.
That made the Rust policy seam stop before allocation and obscured which
allocation owned cleanup after private CDict construction failed.

Move validation, workspace-size query dispatch, allocation, and cleanup
ordering into the Rust bridge. C retains the private workspace-size formula,
workspace/object placement, allocator callbacks, and CDict initialization.
The focused probes cover invalid allocators, allocation failure, creation
failure, and initialization failure with exact callback ordering.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release create_cdict_advanced
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release (665 passed)
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzzer rounds)
This commit is contained in:
ddidderr
2026-07-19 17:10:08 +02:00
parent 5449a4b3c0
commit e6101bb1af
3 changed files with 354 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+87 -49
View File
@@ -1779,14 +1779,24 @@ typedef char ZSTD_rust_init_cdict_state_layout[
&& sizeof(ZSTD_rust_initCDictState) == 19 * sizeof(void*))
? 1 : -1];
typedef void* (*ZSTD_rust_createCDictAdvancedCreate_f)(
typedef int (*ZSTD_rust_createCDictAdvancedValidateCustomMem_f)(void* context);
typedef size_t (*ZSTD_rust_createCDictAdvancedWorkspaceSize_f)(
void* context, size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch);
typedef void* (*ZSTD_rust_createCDictAdvancedAllocate_f)(
void* context, size_t workspaceSize);
typedef void* (*ZSTD_rust_createCDictAdvancedCreate_f)(
void* context, void* workspace, size_t workspaceSize,
size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch);
typedef size_t (*ZSTD_rust_createCDictAdvancedInit_f)(
void* context, void* cdict, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
const ZSTD_CCtx_params* cctxParams);
typedef void (*ZSTD_rust_createCDictAdvancedFreeWorkspace_f)(
void* context, void* workspace);
typedef void (*ZSTD_rust_createCDictAdvancedFree_f)(
void* context, void* cdict);
typedef struct {
@@ -1797,8 +1807,12 @@ typedef struct {
const int* useRowMatchFinder;
U32 exclusionMask;
U32 ldmDefaultWindowLog;
ZSTD_rust_createCDictAdvancedValidateCustomMem_f validateCustomMem;
ZSTD_rust_createCDictAdvancedWorkspaceSize_f workspaceSize;
ZSTD_rust_createCDictAdvancedAllocate_f allocate;
ZSTD_rust_createCDictAdvancedCreate_f create;
ZSTD_rust_createCDictAdvancedInit_f init;
ZSTD_rust_createCDictAdvancedFreeWorkspace_f freeWorkspace;
ZSTD_rust_createCDictAdvancedFree_f free;
} ZSTD_rust_createCDictAdvancedState;
void* ZSTD_rust_createCDictAdvanced(
@@ -1819,14 +1833,22 @@ typedef char ZSTD_rust_create_cdict_advanced_state_layout[
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_createCDictAdvancedState, ldmDefaultWindowLog)
== 5 * sizeof(void*) + sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, create)
&& offsetof(ZSTD_rust_createCDictAdvancedState, validateCustomMem)
== 5 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, init)
&& offsetof(ZSTD_rust_createCDictAdvancedState, workspaceSize)
== 6 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, free)
&& offsetof(ZSTD_rust_createCDictAdvancedState, allocate)
== 7 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, create)
== 8 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, init)
== 9 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, freeWorkspace)
== 10 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, free)
== 11 * sizeof(void*) + 2 * sizeof(U32)
&& sizeof(ZSTD_rust_createCDictAdvancedState)
== 8 * sizeof(void*) + 2 * sizeof(U32))
== 12 * sizeof(void*) + 2 * sizeof(U32))
? 1 : -1];
typedef size_t (*ZSTD_rust_compressBeginResetInternal_f)(
@@ -5544,42 +5566,67 @@ static size_t ZSTD_initCDict_internal(
(int)dictLoadMethod, (int)dictContentType);
}
static ZSTD_CDict*
ZSTD_createCDict_advanced_internal(size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_compressionParameters cParams,
ZSTD_ParamSwitch_e useRowMatchFinder,
int enableDedicatedDictSearch,
ZSTD_customMem customMem)
static int ZSTD_rust_createCDictAdvanced_validateCustomMem(void* context)
{
if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
DEBUGLOG(3, "ZSTD_createCDict_advanced_internal (dictSize=%u)", (unsigned)dictSize);
ZSTD_customMem const* const customMem = (const ZSTD_customMem*)context;
return ((!customMem->customAlloc) ^ (!customMem->customFree)) == 0;
}
{ size_t const workspaceSize =
ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) +
(dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
void* const workspace = ZSTD_customMalloc(workspaceSize, customMem);
ZSTD_cwksp ws;
ZSTD_CDict* cdict;
static size_t ZSTD_rust_createCDictAdvanced_workspaceSize(
void* context, size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch)
{
(void)context;
if (cParams == NULL) return 0;
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ ZSTD_sizeof_matchState(cParams,
(ZSTD_ParamSwitch_e)useRowMatchFinder,
enableDedicatedDictSearch,
/* forCCtx */ 0)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
}
if (!workspace) {
ZSTD_customFree(workspace, customMem);
return NULL;
}
static void* ZSTD_rust_createCDictAdvanced_allocate(
void* context, size_t workspaceSize)
{
return ZSTD_customMalloc(workspaceSize, *(const ZSTD_customMem*)context);
}
ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc);
static void* ZSTD_rust_createCDictAdvanced_create(
void* context, void* workspace, size_t workspaceSize,
size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch)
{
ZSTD_customMem const customMem = *(const ZSTD_customMem*)context;
ZSTD_cwksp ws;
ZSTD_CDict* cdict;
cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
assert(cdict != NULL);
ZSTD_cwksp_move(&cdict->workspace, &ws);
cdict->customMem = customMem;
cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
cdict->useRowMatchFinder = useRowMatchFinder;
return cdict;
}
(void)dictSize;
(void)dictLoadMethod;
(void)cParams;
(void)enableDedicatedDictSearch;
DEBUGLOG(3, "ZSTD_createCDict_advanced_internal (workspaceSize=%u)",
(unsigned)workspaceSize);
if (workspace == NULL) return NULL;
ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc);
cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
if (cdict == NULL) return NULL;
ZSTD_cwksp_move(&cdict->workspace, &ws);
cdict->customMem = customMem;
cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
cdict->useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
return cdict;
}
static void ZSTD_rust_createCDictAdvanced_freeWorkspace(
void* context, void* workspace)
{
ZSTD_customFree(workspace, *(const ZSTD_customMem*)context);
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
@@ -5600,18 +5647,6 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
&cctxParams, customMem);
}
static void* ZSTD_rust_createCDictAdvanced_create(
void* context, size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch)
{
ZSTD_customMem const customMem = *(const ZSTD_customMem*)context;
return ZSTD_createCDict_advanced_internal(
dictSize, (ZSTD_dictLoadMethod_e)dictLoadMethod, *cParams,
(ZSTD_ParamSwitch_e)useRowMatchFinder,
enableDedicatedDictSearch, customMem);
}
static size_t ZSTD_rust_createCDictAdvanced_init(
void* context, void* cdict, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
@@ -5642,7 +5677,6 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
if (originalCctxParams == NULL) return NULL;
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
cctxParams = *originalCctxParams;
state.callbackContext = &customMem;
@@ -5652,8 +5686,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
state.useRowMatchFinder = (const int*)&cctxParams.useRowMatchFinder;
state.exclusionMask = ZSTD_getCParamsExclusionMask();
state.ldmDefaultWindowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
state.validateCustomMem = ZSTD_rust_createCDictAdvanced_validateCustomMem;
state.workspaceSize = ZSTD_rust_createCDictAdvanced_workspaceSize;
state.allocate = ZSTD_rust_createCDictAdvanced_allocate;
state.create = ZSTD_rust_createCDictAdvanced_create;
state.init = ZSTD_rust_createCDictAdvanced_init;
state.freeWorkspace = ZSTD_rust_createCDictAdvanced_freeWorkspace;
state.free = ZSTD_rust_createCDictAdvanced_free;
return (ZSTD_CDict*)ZSTD_rust_createCDictAdvanced(
&state, dict, dictSize,